Environment: Identity Sau Main Dev on web-03
"{\"env\": \"dev\", \"zone\": \"sau\", \"branch\": \"main\", \"db_app\": \"postgresql\", \"service\": \"identity\", \"es_nodes\": 1, \"db_enabled\": true, \"pg_standby\": 1, \"pg_workers\": 1, \"search_app\": \"elasticsearch\", \"description\": \"\", \"iam_enabled\": false, \"worker_1_ip\": \"10.100.1.42\", \"eventbus_app\": \"kafka\", \"es_https_mode\": \"direct\", \"service_es_ip\": \"10.100.1.4\", \"worker_1_fqdn\": \"db-identity-sau-main-dev-postgresql-worker-01.fastorder.com\", \"search_enabled\": true, \"service_app_ip\": \"10.100.1.2\", \"service_obs_ip\": \"10.100.1.18\", \"service_es_fqdn\": \"search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com\", \"service_otlp_ip\": \"10.100.1.30\", \"eventbus_enabled\": true, \"service_app_fqdn\": \"app-identity-sau-main-dev.fastorder.com\", \"service_audit_ip\": \"10.100.1.32\", \"service_obs_fqdn\": \"obs-identity-sau-main-dev.fastorder.com\", \"service_tempo_ip\": \"10.100.1.28\", \"service_endpoints\": \"[{\\\"ip\\\":\\\"10.100.1.3\\\",\\\"fqdn\\\":\\\"app-identity-sau-main-dev.fastorder.com\\\",\\\"service\\\":\\\"app\\\"},{\\\"ip\\\":\\\"10.100.1.5\\\",\\\"fqdn\\\":\\\"search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com\\\",\\\"service\\\":\\\"es_coordinator\\\"},{\\\"ip\\\":\\\"10.100.1.7\\\",\\\"fqdn\\\":\\\"search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com\\\",\\\"service\\\":\\\"es_node_1\\\"},{\\\"ip\\\":\\\"10.100.1.9\\\",\\\"fqdn\\\":\\\"eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com\\\",\\\"service\\\":\\\"kafka_broker_1\\\"},{\\\"ip\\\":\\\"10.100.1.11\\\",\\\"fqdn\\\":\\\"eventbus-identity-sau-main-dev-kafka-connect.fastorder.com\\\",\\\"service\\\":\\\"kafka_connect\\\"},{\\\"ip\\\":\\\"10.100.1.13\\\",\\\"fqdn\\\":\\\"schema-identity-sau-main-dev-kafka-registry.fastorder.com\\\",\\\"service\\\":\\\"kafka_registry\\\"},{\\\"ip\\\":\\\"10.100.1.15\\\",\\\"fqdn\\\":\\\"db-identity-sau-main-dev-postgresql-coordinator.fastorder.com\\\",\\\"service\\\":\\\"pg_coordinator\\\"},{\\\"ip\\\":\\\"10.100.1.17\\\",\\\"fqdn\\\":\\\"db-identity-sau-main-dev-postgresql-bouncer.fastorder.com\\\",\\\"service\\\":\\\"pgbouncer\\\"},{\\\"ip\\\":\\\"10.100.1.19\\\",\\\"fqdn\\\":\\\"obs-identity-sau-main-dev.fastorder.com\\\",\\\"service\\\":\\\"obs\\\"},{\\\"ip\\\":\\\"10.100.1.21\\\",\\\"fqdn\\\":\\\"metrics-identity-sau-main-dev-prometheus.fastorder.com\\\",\\\"service\\\":\\\"metrics\\\"},{\\\"ip\\\":\\\"10.100.1.23\\\",\\\"fqdn\\\":\\\"dashboards-identity-sau-main-dev-grafana.fastorder.com\\\",\\\"service\\\":\\\"dashboards\\\"},{\\\"ip\\\":\\\"10.100.1.25\\\",\\\"fqdn\\\":\\\"alerts-identity-sau-main-dev-alertmanager.fastorder.com\\\",\\\"service\\\":\\\"alerts\\\"},{\\\"ip\\\":\\\"10.100.1.27\\\",\\\"fqdn\\\":\\\"logstore-identity-sau-main-dev-clickhouse.fastorder.com\\\",\\\"service\\\":\\\"logs\\\"},{\\\"ip\\\":\\\"10.100.1.29\\\",\\\"fqdn\\\":\\\"traces-identity-sau-main-dev-tempo.fastorder.com\\\",\\\"service\\\":\\\"traces\\\"},{\\\"ip\\\":\\\"10.100.1.31\\\",\\\"fqdn\\\":\\\"telemetry-identity-sau-main-dev-opentelemetry.fastorder.com\\\",\\\"service\\\":\\\"telemetry\\\"},{\\\"ip\\\":\\\"10.100.1.33\\\",\\\"fqdn\\\":\\\"audit-identity-sau-main-dev.fastorder.com\\\",\\\"service\\\":\\\"audit\\\"},{\\\"ip\\\":\\\"10.100.1.35\\\",\\\"fqdn\\\":\\\"backup-identity-sau-main-dev-db-postgresql.fastorder.com\\\",\\\"service\\\":\\\"backup_pg\\\"},{\\\"ip\\\":\\\"10.100.1.37\\\",\\\"fqdn\\\":\\\"backup-identity-sau-main-dev-eventbus-kafka.fastorder.com\\\",\\\"service\\\":\\\"backup_kafka\\\"},{\\\"ip\\\":\\\"10.100.1.39\\\",\\\"fqdn\\\":\\\"backup-identity-sau-main-dev-search-elasticsearch.fastorder.com\\\",\\\"service\\\":\\\"backup_es\\\"},{\\\"ip\\\":\\\"10.100.1.41\\\",\\\"fqdn\\\":\\\"backup-identity-sau-main-dev-orchestrator.fastorder.com\\\",\\\"service\\\":\\\"backup_orchestrator\\\"}]\", \"service_otlp_fqdn\": \"telemetry-identity-sau-main-dev-opentelemetry.fastorder.com\", \"postgresql_enabled\": true, \"service_audit_fqdn\": \"audit-identity-sau-main-dev.fastorder.com\", \"service_grafana_ip\": \"10.100.1.22\", \"service_tempo_fqdn\": \"traces-identity-sau-main-dev-tempo.fastorder.com\", \"service_backup_es_ip\": \"10.100.1.38\", \"service_backup_pg_ip\": \"10.100.1.34\", \"service_es_node_1_ip\": \"10.100.1.6\", \"service_grafana_fqdn\": \"dashboards-identity-sau-main-dev-grafana.fastorder.com\", \"service_pgbouncer_ip\": \"10.100.1.16\", \"service_prometheus_ip\": \"10.100.1.20\", \"worker_1_standby_1_ip\": \"10.100.1.43\", \"service_backup_es_fqdn\": \"backup-identity-sau-main-dev-search-elasticsearch.fastorder.com\", \"service_backup_pg_fqdn\": \"backup-identity-sau-main-dev-db-postgresql.fastorder.com\", \"service_es_node_1_fqdn\": \"search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com\", \"service_log_backend_ip\": \"10.100.1.26\", \"service_pgbouncer_fqdn\": \"db-identity-sau-main-dev-postgresql-bouncer.fastorder.com\", \"service_alertmanager_ip\": \"10.100.1.24\", \"service_backup_kafka_ip\": \"10.100.1.36\", \"service_prometheus_fqdn\": \"metrics-identity-sau-main-dev-prometheus.fastorder.com\", \"worker_1_standby_1_fqdn\": \"db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com\", \"service_kafka_connect_ip\": \"10.100.1.10\", \"service_log_backend_fqdn\": \"logstore-identity-sau-main-dev-clickhouse.fastorder.com\", \"service_alertmanager_fqdn\": \"alerts-identity-sau-main-dev-alertmanager.fastorder.com\", \"service_backup_kafka_fqdn\": \"backup-identity-sau-main-dev-eventbus-kafka.fastorder.com\", \"service_kafka_broker_1_ip\": \"10.100.1.8\", \"service_kafka_registry_ip\": \"10.100.1.12\", \"service_pg_coordinator_ip\": \"10.100.1.14\", \"service_kafka_connect_fqdn\": \"eventbus-identity-sau-main-dev-kafka-connect.fastorder.com\", \"postgresql_run_verification\": true, \"service_kafka_broker_1_fqdn\": \"eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com\", \"service_kafka_registry_fqdn\": \"schema-identity-sau-main-dev-kafka-registry.fastorder.com\", \"service_pg_coordinator_fqdn\": \"db-identity-sau-main-dev-postgresql-coordinator.fastorder.com\", \"service_backup_orchestrator_ip\": \"10.100.1.40\", \"service_backup_orchestrator_fqdn\": \"backup-identity-sau-main-dev-orchestrator.fastorder.com\"}"
This job encountered an error. You can restart from the failed step.
This job has been restarted. You are viewing an older attempt. The logs and status shown below are from the latest retry.
This job failed at one of the steps below. You can resume from where it failed to save time and avoid re-running successful steps.
[1m════════════════════════════════════════════════════════════════[0m
[1m FastOrder Pre-Flight Validation Checks[0m
[1m════════════════════════════════════════════════════════════════[0m
[0;34m[INFO][0m Checking SSH connectivity to target host...
[0;32m[✓][0m Target is localhost, skipping SSH check
[0;34m[INFO][0m Checking available disk space...
[1;33m[⚠][0m Disk space limited: 22GB available (recommended: 50GB)
→ PostgreSQL + Elasticsearch may experience space pressure
[0;34m[INFO][0m Checking available memory...
[1;33m[⚠][0m Memory low: 7GB (minimum: 4GB, recommended: 16GB)
→ Suitable for development/testing only
→ Reduce component counts: use 1 ES node, 1 PG worker, minimal standby nodes
→ Production environments require 16GB+
[0;34m[INFO][0m Checking critical port availability...
[0;32m[✓][0m Port 5432 in use on specific IP (10.100.1.189:5432) - OK, can use different IP
[0;32m[✓][0m Port 9200 in use on specific IP ([::ffff:10.100.1.179]) - OK, can use different IP
[0;32m[✓][0m Port 9300 in use on specific IP ([::ffff:10.100.1.179]) - OK, can use different IP
[0;32m[✓][0m Port 9092 available (Kafka)
[0;32m[✓][0m Port 2181 available (Zookeeper)
[0;34m[INFO][0m Checking DNS resolution...
[0;32m[✓][0m DNS resolution working: google.com
[0;32m[✓][0m DNS resolution working: github.com
[0;32m[✓][0m DNS resolution working: archive.ubuntu.com
[0;34m[INFO][0m Checking required system commands...
[0;32m[✓][0m Command available: curl
[0;32m[✓][0m Command available: wget
[0;32m[✓][0m Command available: git
[0;32m[✓][0m Command available: sudo
[0;32m[✓][0m Command available: systemctl
[0;32m[✓][0m Command available: apt-get
[0;34m[INFO][0m Checking current system load...
[1;33m[⚠][0m System load elevated: 3.54 (4 CPUs)
→ Provisioning may be slower than expected
[0;34m[INFO][0m Checking for existing environment conflicts...
[0;32m[✓][0m No conflicting services found for: identity-uae-main-dev
[1m════════════════════════════════════════════════════════════════[0m
[1m Pre-Flight Check Summary[0m
[1m════════════════════════════════════════════════════════════════[0m
[1;33m[⚠][0m 3 warning(s) detected
⚠️ Environment can proceed with caution
Review warnings above and consider remediation
[INFO] Using web-provided environment: identity-sau-main-dev
[INFO] Auto-creating state directory for identity-sau-main-dev...
[ OK ] Created topology.json for identity-sau-main-dev
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=10.100.1.51)
[0;36m[2026-01-02_06:37:27][0m Starting Terraform provisioning step
[0;36m[2026-01-02_06:37:27][0m Service: identity
[0;36m[2026-01-02_06:37:27][0m Zone: sau
[0;36m[2026-01-02_06:37:27][0m Environment: dev
[0;36m[2026-01-02_06:37:27][0m Resource: web-03
[0;36m[2026-01-02_06:37:27][0m Terraform binary: /home/ab/bin/terraform
[0;36m[2026-01-02_06:37:27][0m HOME: /home/www-data
[0;36m[2026-01-02_06:37:27][0m AWS Config: /home/ab/.aws/config
[0;36m[2026-01-02_06:37:27][0m AWS Credentials: /home/ab/.aws/credentials
[0;36m[2026-01-02_06:37:27][0m Terraform directory: /opt/fastorder/cli/terraform/examples/citus-production
[0;36m[2026-01-02_06:37:27][0m Running terraform init...
[0m[1mInitializing the backend...[0m
[0m[1mUpgrading modules...[0m
- citus_cluster in ../../modules/citus_cluster
[0m[1mInitializing provider plugins...[0m
- Finding hashicorp/aws versions matching "~> 5.0"...
- Using previously-installed hashicorp/aws v5.100.0
[0m[1m[32mTerraform has been successfully initialized![0m[32m[0m
[0m[32m
You may now begin working with Terraform. Try running "terraform plan" to see
any changes that are required for your infrastructure. All Terraform commands
should now work.
If you ever set or change modules or backend configuration for Terraform,
rerun this command to reinitialize your working directory. If you forget, other
commands will detect it and remind you to do so if necessary.[0m
[0;32m[2026-01-02_06:37:39] ✓[0m Terraform init succeeded
[0;36m[2026-01-02_06:37:39][0m Running terraform validate...
[32m[1mSuccess![0m The configuration is valid.
[0m
[0;32m[2026-01-02_06:37:47] ✓[0m Terraform validate succeeded
[0;36m[2026-01-02_06:37:47][0m Running terraform plan...
[0m[1mmodule.citus_cluster.data.aws_caller_identity.current: Reading...[0m[0m
[0m[1mmodule.citus_cluster.data.aws_caller_identity.current: Read complete after 0s [id=464621692046][0m
Terraform used the selected providers to generate the following execution
plan. Resource actions are indicated with the following symbols:
[32m+[0m create[0m
Terraform will perform the following actions:
[1m # module.citus_cluster.aws_iam_instance_profile.citus[0m will be created
[0m [32m+[0m[0m resource "aws_iam_instance_profile" "citus" {
[32m+[0m[0m arn = (known after apply)
[32m+[0m[0m create_date = (known after apply)
[32m+[0m[0m id = (known after apply)
[32m+[0m[0m name = (known after apply)
[32m+[0m[0m name_prefix = "citus-prod-"
[32m+[0m[0m path = "/"
[32m+[0m[0m role = (known after apply)
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-prod"
}
[32m+[0m[0m tags_all = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "ManagedBy" = "Terraform"
[32m+[0m[0m "Name" = "citus-prod"
[32m+[0m[0m "Owner" = "Platform Team"
[32m+[0m[0m "Project" = "FastOrder"
}
[32m+[0m[0m unique_id = (known after apply)
}
[1m # module.citus_cluster.aws_iam_role.citus[0m will be created
[0m [32m+[0m[0m resource "aws_iam_role" "citus" {
[32m+[0m[0m arn = (known after apply)
[32m+[0m[0m assume_role_policy = jsonencode(
{
[32m+[0m[0m Statement = [
[32m+[0m[0m {
[32m+[0m[0m Action = "sts:AssumeRole"
[32m+[0m[0m Effect = "Allow"
[32m+[0m[0m Principal = {
[32m+[0m[0m Service = "ec2.amazonaws.com"
}
},
]
[32m+[0m[0m Version = "2012-10-17"
}
)
[32m+[0m[0m create_date = (known after apply)
[32m+[0m[0m force_detach_policies = false
[32m+[0m[0m id = (known after apply)
[32m+[0m[0m managed_policy_arns = (known after apply)
[32m+[0m[0m max_session_duration = 3600
[32m+[0m[0m name = (known after apply)
[32m+[0m[0m name_prefix = "citus-prod-"
[32m+[0m[0m path = "/"
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-prod"
}
[32m+[0m[0m tags_all = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "ManagedBy" = "Terraform"
[32m+[0m[0m "Name" = "citus-prod"
[32m+[0m[0m "Owner" = "Platform Team"
[32m+[0m[0m "Project" = "FastOrder"
}
[32m+[0m[0m unique_id = (known after apply)
}
[1m # module.citus_cluster.aws_iam_role_policy.secrets_manager[0][0m will be created
[0m [32m+[0m[0m resource "aws_iam_role_policy" "secrets_manager" {
[32m+[0m[0m id = (known after apply)
[32m+[0m[0m name = (known after apply)
[32m+[0m[0m name_prefix = "secrets-access-"
[32m+[0m[0m policy = jsonencode(
{
[32m+[0m[0m Statement = [
[32m+[0m[0m {
[32m+[0m[0m Action = [
[32m+[0m[0m "secretsmanager:GetSecretValue",
[32m+[0m[0m "secretsmanager:DescribeSecret",
]
[32m+[0m[0m Effect = "Allow"
[32m+[0m[0m Resource = "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/db/web/ksa/main/dev/postgresqladmin/ksa/prod*"
},
]
[32m+[0m[0m Version = "2012-10-17"
}
)
[32m+[0m[0m role = (known after apply)
}
[1m # module.citus_cluster.aws_iam_role_policy_attachment.cloudwatch[0m will be created
[0m [32m+[0m[0m resource "aws_iam_role_policy_attachment" "cloudwatch" {
[32m+[0m[0m id = (known after apply)
[32m+[0m[0m policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
[32m+[0m[0m role = (known after apply)
}
[1m # module.citus_cluster.aws_iam_role_policy_attachment.ssm[0m will be created
[0m [32m+[0m[0m resource "aws_iam_role_policy_attachment" "ssm" {
[32m+[0m[0m id = (known after apply)
[32m+[0m[0m policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
[32m+[0m[0m role = (known after apply)
}
[1m # module.citus_cluster.aws_instance.coordinator[0m will be created
[0m [32m+[0m[0m resource "aws_instance" "coordinator" {
[32m+[0m[0m ami = "ami-0b2aae5f4283c0df2"
[32m+[0m[0m arn = (known after apply)
[32m+[0m[0m associate_public_ip_address = (known after apply)
[32m+[0m[0m availability_zone = (known after apply)
[32m+[0m[0m cpu_core_count = (known after apply)
[32m+[0m[0m cpu_threads_per_core = (known after apply)
[32m+[0m[0m disable_api_stop = (known after apply)
[32m+[0m[0m disable_api_termination = (known after apply)
[32m+[0m[0m ebs_optimized = (known after apply)
[32m+[0m[0m enable_primary_ipv6 = (known after apply)
[32m+[0m[0m get_password_data = false
[32m+[0m[0m host_id = (known after apply)
[32m+[0m[0m host_resource_group_arn = (known after apply)
[32m+[0m[0m iam_instance_profile = (known after apply)
[32m+[0m[0m id = (known after apply)
[32m+[0m[0m instance_initiated_shutdown_behavior = (known after apply)
[32m+[0m[0m instance_lifecycle = (known after apply)
[32m+[0m[0m instance_state = (known after apply)
[32m+[0m[0m instance_type = "r6i.2xlarge"
[32m+[0m[0m ipv6_address_count = (known after apply)
[32m+[0m[0m ipv6_addresses = (known after apply)
[32m+[0m[0m key_name = (known after apply)
[32m+[0m[0m monitoring = (known after apply)
[32m+[0m[0m outpost_arn = (known after apply)
[32m+[0m[0m password_data = (known after apply)
[32m+[0m[0m placement_group = (known after apply)
[32m+[0m[0m placement_partition_number = (known after apply)
[32m+[0m[0m primary_network_interface_id = (known after apply)
[32m+[0m[0m private_dns = (known after apply)
[32m+[0m[0m private_ip = (known after apply)
[32m+[0m[0m public_dns = (known after apply)
[32m+[0m[0m public_ip = (known after apply)
[32m+[0m[0m secondary_private_ips = (known after apply)
[32m+[0m[0m security_groups = (known after apply)
[32m+[0m[0m source_dest_check = true
[32m+[0m[0m spot_instance_request_id = (known after apply)
[32m+[0m[0m subnet_id = "subnet-0a1f5a9a74ed030cf"
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-coordinator-prod"
[32m+[0m[0m "Role" = "coordinator"
[32m+[0m[0m "Service" = "citus"
}
[32m+[0m[0m tags_all = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "ManagedBy" = "Terraform"
[32m+[0m[0m "Name" = "citus-coordinator-prod"
[32m+[0m[0m "Owner" = "Platform Team"
[32m+[0m[0m "Project" = "FastOrder"
[32m+[0m[0m "Role" = "coordinator"
[32m+[0m[0m "Service" = "citus"
}
[32m+[0m[0m tenancy = (known after apply)
[32m+[0m[0m user_data = "2a9e41ea765dcf3b3046ee10d2f458c18f00e430"
[32m+[0m[0m user_data_base64 = (known after apply)
[32m+[0m[0m user_data_replace_on_change = false
[32m+[0m[0m vpc_security_group_ids = (known after apply)
[32m+[0m[0m ebs_block_device {
[32m+[0m[0m delete_on_termination = false
[32m+[0m[0m device_name = "/dev/sdf"
[32m+[0m[0m encrypted = true
[32m+[0m[0m iops = 3000
[32m+[0m[0m kms_key_id = (known after apply)
[32m+[0m[0m snapshot_id = (known after apply)
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-coordinator-prod-data"
}
[32m+[0m[0m tags_all = (known after apply)
[32m+[0m[0m throughput = 125
[32m+[0m[0m volume_id = (known after apply)
[32m+[0m[0m volume_size = 500
[32m+[0m[0m volume_type = "gp3"
}
[32m+[0m[0m root_block_device {
[32m+[0m[0m delete_on_termination = false
[32m+[0m[0m device_name = (known after apply)
[32m+[0m[0m encrypted = true
[32m+[0m[0m iops = (known after apply)
[32m+[0m[0m kms_key_id = (known after apply)
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-coordinator-prod-root"
}
[32m+[0m[0m tags_all = (known after apply)
[32m+[0m[0m throughput = (known after apply)
[32m+[0m[0m volume_id = (known after apply)
[32m+[0m[0m volume_size = 100
[32m+[0m[0m volume_type = "gp3"
}
}
[1m # module.citus_cluster.aws_instance.workers[0][0m will be created
[0m [32m+[0m[0m resource "aws_instance" "workers" {
[32m+[0m[0m ami = "ami-0b2aae5f4283c0df2"
[32m+[0m[0m arn = (known after apply)
[32m+[0m[0m associate_public_ip_address = (known after apply)
[32m+[0m[0m availability_zone = (known after apply)
[32m+[0m[0m cpu_core_count = (known after apply)
[32m+[0m[0m cpu_threads_per_core = (known after apply)
[32m+[0m[0m disable_api_stop = (known after apply)
[32m+[0m[0m disable_api_termination = (known after apply)
[32m+[0m[0m ebs_optimized = (known after apply)
[32m+[0m[0m enable_primary_ipv6 = (known after apply)
[32m+[0m[0m get_password_data = false
[32m+[0m[0m host_id = (known after apply)
[32m+[0m[0m host_resource_group_arn = (known after apply)
[32m+[0m[0m iam_instance_profile = (known after apply)
[32m+[0m[0m id = (known after apply)
[32m+[0m[0m instance_initiated_shutdown_behavior = (known after apply)
[32m+[0m[0m instance_lifecycle = (known after apply)
[32m+[0m[0m instance_state = (known after apply)
[32m+[0m[0m instance_type = "r6i.2xlarge"
[32m+[0m[0m ipv6_address_count = (known after apply)
[32m+[0m[0m ipv6_addresses = (known after apply)
[32m+[0m[0m key_name = (known after apply)
[32m+[0m[0m monitoring = (known after apply)
[32m+[0m[0m outpost_arn = (known after apply)
[32m+[0m[0m password_data = (known after apply)
[32m+[0m[0m placement_group = (known after apply)
[32m+[0m[0m placement_partition_number = (known after apply)
[32m+[0m[0m primary_network_interface_id = (known after apply)
[32m+[0m[0m private_dns = (known after apply)
[32m+[0m[0m private_ip = (known after apply)
[32m+[0m[0m public_dns = (known after apply)
[32m+[0m[0m public_ip = (known after apply)
[32m+[0m[0m secondary_private_ips = (known after apply)
[32m+[0m[0m security_groups = (known after apply)
[32m+[0m[0m source_dest_check = true
[32m+[0m[0m spot_instance_request_id = (known after apply)
[32m+[0m[0m subnet_id = "subnet-0a1f5a9a74ed030cf"
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-worker-0-prod"
[32m+[0m[0m "Role" = "worker"
[32m+[0m[0m "Service" = "citus"
[32m+[0m[0m "WorkerIndex" = "0"
}
[32m+[0m[0m tags_all = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "ManagedBy" = "Terraform"
[32m+[0m[0m "Name" = "citus-worker-0-prod"
[32m+[0m[0m "Owner" = "Platform Team"
[32m+[0m[0m "Project" = "FastOrder"
[32m+[0m[0m "Role" = "worker"
[32m+[0m[0m "Service" = "citus"
[32m+[0m[0m "WorkerIndex" = "0"
}
[32m+[0m[0m tenancy = (known after apply)
[32m+[0m[0m user_data = "7b4bd87c9982aab7fa463c8d12e99399661f8bde"
[32m+[0m[0m user_data_base64 = (known after apply)
[32m+[0m[0m user_data_replace_on_change = false
[32m+[0m[0m vpc_security_group_ids = (known after apply)
[32m+[0m[0m ebs_block_device {
[32m+[0m[0m delete_on_termination = false
[32m+[0m[0m device_name = "/dev/sdf"
[32m+[0m[0m encrypted = true
[32m+[0m[0m iops = 3000
[32m+[0m[0m kms_key_id = (known after apply)
[32m+[0m[0m snapshot_id = (known after apply)
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-worker-0-prod-data"
}
[32m+[0m[0m tags_all = (known after apply)
[32m+[0m[0m throughput = 125
[32m+[0m[0m volume_id = (known after apply)
[32m+[0m[0m volume_size = 500
[32m+[0m[0m volume_type = "gp3"
}
[32m+[0m[0m root_block_device {
[32m+[0m[0m delete_on_termination = false
[32m+[0m[0m device_name = (known after apply)
[32m+[0m[0m encrypted = true
[32m+[0m[0m iops = (known after apply)
[32m+[0m[0m kms_key_id = (known after apply)
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-worker-0-prod-root"
}
[32m+[0m[0m tags_all = (known after apply)
[32m+[0m[0m throughput = (known after apply)
[32m+[0m[0m volume_id = (known after apply)
[32m+[0m[0m volume_size = 100
[32m+[0m[0m volume_type = "gp3"
}
}
[1m # module.citus_cluster.aws_instance.workers[1][0m will be created
[0m [32m+[0m[0m resource "aws_instance" "workers" {
[32m+[0m[0m ami = "ami-0b2aae5f4283c0df2"
[32m+[0m[0m arn = (known after apply)
[32m+[0m[0m associate_public_ip_address = (known after apply)
[32m+[0m[0m availability_zone = (known after apply)
[32m+[0m[0m cpu_core_count = (known after apply)
[32m+[0m[0m cpu_threads_per_core = (known after apply)
[32m+[0m[0m disable_api_stop = (known after apply)
[32m+[0m[0m disable_api_termination = (known after apply)
[32m+[0m[0m ebs_optimized = (known after apply)
[32m+[0m[0m enable_primary_ipv6 = (known after apply)
[32m+[0m[0m get_password_data = false
[32m+[0m[0m host_id = (known after apply)
[32m+[0m[0m host_resource_group_arn = (known after apply)
[32m+[0m[0m iam_instance_profile = (known after apply)
[32m+[0m[0m id = (known after apply)
[32m+[0m[0m instance_initiated_shutdown_behavior = (known after apply)
[32m+[0m[0m instance_lifecycle = (known after apply)
[32m+[0m[0m instance_state = (known after apply)
[32m+[0m[0m instance_type = "r6i.2xlarge"
[32m+[0m[0m ipv6_address_count = (known after apply)
[32m+[0m[0m ipv6_addresses = (known after apply)
[32m+[0m[0m key_name = (known after apply)
[32m+[0m[0m monitoring = (known after apply)
[32m+[0m[0m outpost_arn = (known after apply)
[32m+[0m[0m password_data = (known after apply)
[32m+[0m[0m placement_group = (known after apply)
[32m+[0m[0m placement_partition_number = (known after apply)
[32m+[0m[0m primary_network_interface_id = (known after apply)
[32m+[0m[0m private_dns = (known after apply)
[32m+[0m[0m private_ip = (known after apply)
[32m+[0m[0m public_dns = (known after apply)
[32m+[0m[0m public_ip = (known after apply)
[32m+[0m[0m secondary_private_ips = (known after apply)
[32m+[0m[0m security_groups = (known after apply)
[32m+[0m[0m source_dest_check = true
[32m+[0m[0m spot_instance_request_id = (known after apply)
[32m+[0m[0m subnet_id = "subnet-02c930351cde1e9c3"
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-worker-1-prod"
[32m+[0m[0m "Role" = "worker"
[32m+[0m[0m "Service" = "citus"
[32m+[0m[0m "WorkerIndex" = "1"
}
[32m+[0m[0m tags_all = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "ManagedBy" = "Terraform"
[32m+[0m[0m "Name" = "citus-worker-1-prod"
[32m+[0m[0m "Owner" = "Platform Team"
[32m+[0m[0m "Project" = "FastOrder"
[32m+[0m[0m "Role" = "worker"
[32m+[0m[0m "Service" = "citus"
[32m+[0m[0m "WorkerIndex" = "1"
}
[32m+[0m[0m tenancy = (known after apply)
[32m+[0m[0m user_data = "7b4bd87c9982aab7fa463c8d12e99399661f8bde"
[32m+[0m[0m user_data_base64 = (known after apply)
[32m+[0m[0m user_data_replace_on_change = false
[32m+[0m[0m vpc_security_group_ids = (known after apply)
[32m+[0m[0m ebs_block_device {
[32m+[0m[0m delete_on_termination = false
[32m+[0m[0m device_name = "/dev/sdf"
[32m+[0m[0m encrypted = true
[32m+[0m[0m iops = 3000
[32m+[0m[0m kms_key_id = (known after apply)
[32m+[0m[0m snapshot_id = (known after apply)
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-worker-1-prod-data"
}
[32m+[0m[0m tags_all = (known after apply)
[32m+[0m[0m throughput = 125
[32m+[0m[0m volume_id = (known after apply)
[32m+[0m[0m volume_size = 500
[32m+[0m[0m volume_type = "gp3"
}
[32m+[0m[0m root_block_device {
[32m+[0m[0m delete_on_termination = false
[32m+[0m[0m device_name = (known after apply)
[32m+[0m[0m encrypted = true
[32m+[0m[0m iops = (known after apply)
[32m+[0m[0m kms_key_id = (known after apply)
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-worker-1-prod-root"
}
[32m+[0m[0m tags_all = (known after apply)
[32m+[0m[0m throughput = (known after apply)
[32m+[0m[0m volume_id = (known after apply)
[32m+[0m[0m volume_size = 100
[32m+[0m[0m volume_type = "gp3"
}
}
[1m # module.citus_cluster.aws_security_group.citus[0m will be created
[0m [32m+[0m[0m resource "aws_security_group" "citus" {
[32m+[0m[0m arn = (known after apply)
[32m+[0m[0m description = "Security group for Citus cluster"
[32m+[0m[0m egress = [
[32m+[0m[0m {
[32m+[0m[0m cidr_blocks = [
[32m+[0m[0m "0.0.0.0/0",
]
[32m+[0m[0m description = "Allow all outbound"
[32m+[0m[0m from_port = 0
[32m+[0m[0m ipv6_cidr_blocks = []
[32m+[0m[0m prefix_list_ids = []
[32m+[0m[0m protocol = "-1"
[32m+[0m[0m security_groups = []
[32m+[0m[0m self = false
[32m+[0m[0m to_port = 0
},
]
[32m+[0m[0m id = (known after apply)
[32m+[0m[0m ingress = [
[32m+[0m[0m {
[32m+[0m[0m cidr_blocks = [
[32m+[0m[0m "10.0.0.0/8",
]
[32m+[0m[0m description = "PgBouncer access"
[32m+[0m[0m from_port = 6432
[32m+[0m[0m ipv6_cidr_blocks = []
[32m+[0m[0m prefix_list_ids = []
[32m+[0m[0m protocol = "tcp"
[32m+[0m[0m security_groups = []
[32m+[0m[0m self = false
[32m+[0m[0m to_port = 6432
},
[32m+[0m[0m {
[32m+[0m[0m cidr_blocks = [
[32m+[0m[0m "10.0.0.0/8",
]
[32m+[0m[0m description = "PostgreSQL access"
[32m+[0m[0m from_port = 5432
[32m+[0m[0m ipv6_cidr_blocks = []
[32m+[0m[0m prefix_list_ids = []
[32m+[0m[0m protocol = "tcp"
[32m+[0m[0m security_groups = []
[32m+[0m[0m self = false
[32m+[0m[0m to_port = 5432
},
[32m+[0m[0m {
[32m+[0m[0m cidr_blocks = [
[32m+[0m[0m "10.0.0.0/8",
]
[32m+[0m[0m description = "SSH access"
[32m+[0m[0m from_port = 22
[32m+[0m[0m ipv6_cidr_blocks = []
[32m+[0m[0m prefix_list_ids = []
[32m+[0m[0m protocol = "tcp"
[32m+[0m[0m security_groups = []
[32m+[0m[0m self = false
[32m+[0m[0m to_port = 22
},
[32m+[0m[0m {
[32m+[0m[0m cidr_blocks = []
[32m+[0m[0m description = "Internal cluster communication"
[32m+[0m[0m from_port = 0
[32m+[0m[0m ipv6_cidr_blocks = []
[32m+[0m[0m prefix_list_ids = []
[32m+[0m[0m protocol = "tcp"
[32m+[0m[0m security_groups = []
[32m+[0m[0m self = true
[32m+[0m[0m to_port = 65535
},
]
[32m+[0m[0m name = (known after apply)
[32m+[0m[0m name_prefix = "citus-prod-"
[32m+[0m[0m owner_id = (known after apply)
[32m+[0m[0m revoke_rules_on_delete = false
[32m+[0m[0m tags = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "Name" = "citus-prod"
[32m+[0m[0m "Service" = "citus"
}
[32m+[0m[0m tags_all = {
[32m+[0m[0m "Backup" = "Required"
[32m+[0m[0m "CostCenter" = "Platform"
[32m+[0m[0m "Environment" = "prod"
[32m+[0m[0m "ManagedBy" = "Terraform"
[32m+[0m[0m "Name" = "citus-prod"
[32m+[0m[0m "Owner" = "Platform Team"
[32m+[0m[0m "Project" = "FastOrder"
[32m+[0m[0m "Service" = "citus"
}
[32m+[0m[0m vpc_id = "vpc-0af7da1e7d94d62bd"
}
[1mPlan:[0m 9 to add, 0 to change, 0 to destroy.
[0m
Changes to Outputs:
[32m+[0m[0m connection_string = (sensitive value)
[32m+[0m[0m coordinator_ip = (known after apply)
[32m+[0m[0m worker_ips = [
[32m+[0m[0m (known after apply),
[32m+[0m[0m (known after apply),
]
[90m
─────────────────────────────────────────────────────────────────────────────[0m
Saved the plan to: tfplan
To perform exactly these actions, run the following command to apply:
terraform apply "tfplan"
[0;32m[2026-01-02_06:37:56] ✓[0m Terraform plan succeeded
[0;36m[2026-01-02_06:37:56][0m Generating plan JSON...
[0;32m[2026-01-02_06:38:04] ✓[0m Terraform provisioning step completed successfully
Next step: Review the plan and apply with 'terraform apply tfplan'
[INFO] FastOrder Environment Preparation
[INFO] Service: identity
[INFO] Zone: sau
[INFO] Environment: dev
[INFO] Branch: main
[INFO] State Directory: /opt/fastorder/bash/scripts/env_app_setup/state
[INFO] Library: /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator
[INFO] IP: 142.93.238.16 (specified)
[INFO] Creating environment using fo-env...
[INFO] Creating new FastOrder environment (v1 topology)
[INFO] Generated environment ID: identity-sau-main-dev
[INFO] Using provided IP: 142.93.238.16
[INFO] Allocated interface: eth0:16
[INFO] Configuring network interface for VM IP: 142.93.238.16
[INFO] VM IP 142.93.238.16 is already configured on eth0:16
[CONFIG] No web configuration found for environment: identity-sau-main-dev
[CONFIG] Using defaults: ES_NODES=1, PG_WORKERS=1
[INFO] Service enabled flags: db=yes, eventbus=yes, search=yes
[ OK ] Created topology.json at /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[ OK ] Generated overlay configurations in /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/generated/
[ OK ] Updated environments.json
[ OK ] Updated setup.json
[ OK ] Environment created successfully!
[INFO]
[INFO] Environment Details:
[INFO] ID: identity-sau-main-dev
[INFO] Service: identity
[INFO] zone: sau
[INFO] Environment: dev
[INFO] Branch: main
[INFO] IP: 142.93.238.16
[INFO] Interface: eth0:16
[INFO]
[INFO] Configuration files:
[INFO] Topology: /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[INFO] Generated: /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/generated/*.env
[INFO] Overrides: /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/overrides/*.env
[INFO]
[INFO] To use this environment:
[INFO] export ENV_ID="identity-sau-main-dev"
[INFO] source /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh
[INFO] init_environment
[ OK ] Environment preparation completed successfully!
[INFO] Creating topology from web form submission...
[INFO] Loaded from topology.json: identity-sau-main-dev
[0;32m[2026-01-02 06:38:09][0m Loaded environment: identity-sau-main-dev
[0;32m[2026-01-02 06:38:09][0m Service: identity, Zone: sau, Branch: main, Env: dev
[0;32m[2026-01-02 06:38:09][0m VM IP: 142.93.238.16, Interface: eth0:16
[0;32m[2026-01-02 06:38:09][0m Elasticsearch Nodes: 1, PostgreSQL Workers: 1
[0;32m[2026-01-02 06:38:09][0m PostgreSQL HA Nodes: 1, Citus Enabled: yes
[ OK ] Environment initialized successfully (mode: general)
[INFO] Creating topology.json from web form submission...
[INFO] DEBUG: Service enabled flags...
[INFO] DB_ENABLED=yes
[INFO] EVENTBUS_ENABLED=yes
[INFO] SEARCH_ENABLED=yes
[INFO] DEBUG: Checking for form submission variables...
[INFO] service_es_ip=10.100.1.4
[INFO] service_es_fqdn=search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com
[INFO] service_pg_coordinator_ip=10.100.1.14
[WARN] IP 10.100.1.4 is already allocated, allocating new IP for search
/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/common.sh: line 261: echo: write error: Broken pipe
[INFO] Adding search: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com (10.100.1.94) [reallocated from 10.100.1.4]
[WARN] IP 10.100.1.6 is already allocated, allocating new IP for search-node-01
/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/common.sh: line 261: echo: write error: Broken pipe
[INFO] Adding search-node-01: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103) [reallocated from 10.100.1.6]
[WARN] IP 10.100.1.8 is already allocated, allocating new IP for eventbus-broker-01
[INFO] Adding eventbus-broker-01: eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com (10.100.1.199) [reallocated from 10.100.1.8]
[WARN] IP 10.100.1.10 is already allocated, allocating new IP for eventbus-connect
[INFO] Adding eventbus-connect: eventbus-identity-sau-main-dev-kafka-connect.fastorder.com (10.100.1.201) [reallocated from 10.100.1.10]
[WARN] IP 10.100.1.12 is already allocated, allocating new IP for schema-registry
[INFO] Adding schema-registry: schema-identity-sau-main-dev-kafka-registry.fastorder.com (10.100.1.202) [reallocated from 10.100.1.12]
[WARN] IP 10.100.1.14 is already allocated, allocating new IP for pg-coordinator
[INFO] Adding pg-coordinator: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com (10.100.1.203) [reallocated from 10.100.1.14]
[WARN] IP 10.100.1.16 is already allocated, allocating new IP for pgbouncer
[INFO] Adding pgbouncer: db-identity-sau-main-dev-postgresql-bouncer.fastorder.com (10.100.1.204) [reallocated from 10.100.1.16]
[WARN] IP 10.100.1.18 is already allocated, allocating new IP for obs
/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/common.sh: line 261: echo: write error: Broken pipe
[INFO] Adding obs: obs-identity-sau-main-dev.fastorder.com (10.100.1.51) [reallocated from 10.100.1.18]
[ OK ] Topology created from form data
[INFO] Applications registered:
✓ eventbus-broker-01: eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com (10.100.1.199)
✓ eventbus-connect: eventbus-identity-sau-main-dev-kafka-connect.fastorder.com (10.100.1.201)
✓ obs: obs-identity-sau-main-dev.fastorder.com (10.100.1.51)
✓ pg-coordinator: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com (10.100.1.203)
✓ pgbouncer: db-identity-sau-main-dev-postgresql-bouncer.fastorder.com (10.100.1.204)
✓ schema-registry: schema-identity-sau-main-dev-kafka-registry.fastorder.com (10.100.1.202)
✓ search: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com (10.100.1.94)
✓ search-node-01: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
[ OK ] Topology created from form data
[INFO] Next steps:
[INFO] 1. Review the generated topology.json and configurations
[INFO] 2. Customize overrides/*.env files if needed
[INFO] 3. Run subsequent installation steps (02-install-postgresql, etc.)
[INFO] To use this environment in other scripts:
[INFO] export ENV_ID="$(fo-env list | tail -n1 | awk '{print $1}')"
[INFO] source /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh
[INFO] init_environment
⏳ This step is pending and will execute after the previous steps complete successfully.
Loading logs...
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m 🚀 OBSERVABILITY CELL PROVISIONING STARTED
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Script: 02-observability-cell/run.sh
[0;34m[INFO][0m Timestamp: 2026-01-02 06:38:23 UTC
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Ensuring correct permissions for observability deployment...
[2026-01-02 06:38:23 UTC] USER=www-data EUID=0 PID=1552489 ACTION=fsop ARGS=chown www-data:www-data /var/log/fastorder
[2026-01-02 06:38:23 UTC] USER=www-data EUID=0 PID=1552498 ACTION=fsop ARGS=touch /var/log/fastorder/provisioning-elevated.log
[2026-01-02 06:38:23 UTC] USER=www-data EUID=0 PID=1552507 ACTION=fsop ARGS=chmod 666 /var/log/fastorder/provisioning-elevated.log
[2026-01-02 06:38:23 UTC] USER=www-data EUID=0 PID=1552516 ACTION=fsop ARGS=chown www-data:www-data /var/log/fastorder/provisioning-elevated.log
[0;32m[OK][0m Log directory: /var/log/fastorder (775)
[0;32m[OK][0m Log file: provisioning-elevated.log (666)
[2026-01-02 06:38:23 UTC] USER=www-data EUID=0 PID=1552525 ACTION=fsop ARGS=chmod 775 /opt/fastorder/bash/scripts/env_app_setup/state
[0;32m[OK][0m State directory: 775
[2026-01-02 06:38:23 UTC] USER=www-data EUID=0 PID=1552535 ACTION=fsop ARGS=mkdir -p /etc/fastorder/observability/certs
[2026-01-02 06:38:23 UTC] USER=www-data EUID=0 PID=1552544 ACTION=fsop ARGS=chmod 750 /etc/fastorder/observability/certs
[0;32m[OK][0m Cert directory: /etc/fastorder/observability/certs (750 - secure)
[0;32m[OK][0m Lib scripts: executable (755)
[0;32m[OK][0m All deployment scripts: executable (755)
[0;32m[OK][0m All directories: accessible (755)
[0;32m[OK][0m ✅ All permissions verified and fixed
[0;34m[CREDS][0m Using AWS credentials from: /var/www/.aws/credentials
[0;34m[CREDS][0m Credential management library loaded (region: me-central-1)
[INFO] Using web-provided environment: identity-sau-main-dev
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
═══════════════════════════════════════════════════════════════════════════════
OBSERVABILITY CELL PROVISIONING
═══════════════════════════════════════════════════════════════════════════════
[INFO] Application Cell: identity-sau-main-dev
[INFO] Observability Cell: obs-identity-sau-main-dev
[INFO] Service: identity | Zone: sau | Env: dev
[INFO] Step 1/10: Provisioning network infrastructure...
[INFO] Using existing IP for obs: 10.100.1.51
[INFO] Allocated new IP for metrics: 10.100.1.205
[2026-01-02 06:38:26 UTC] USER=www-data EUID=0 PID=1553023 ACTION=fsop ARGS=cp /tmp/tmp.OqwNn440sj /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[INFO] Allocated new IP for dashboards: 10.100.1.206
[2026-01-02 06:38:26 UTC] USER=www-data EUID=0 PID=1553040 ACTION=fsop ARGS=cp /tmp/tmp.kzdNKV6ePM /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[INFO] Allocated new IP for logstore: 10.100.1.207
[2026-01-02 06:38:26 UTC] USER=www-data EUID=0 PID=1553075 ACTION=fsop ARGS=cp /tmp/tmp.BXODmhvpso /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[INFO] Allocated new IP for traces: 10.100.1.208
[2026-01-02 06:38:26 UTC] USER=www-data EUID=0 PID=1553099 ACTION=fsop ARGS=cp /tmp/tmp.Nyt4HSGiN0 /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[INFO] Allocated new IP for alerts: 10.100.1.209
[2026-01-02 06:38:26 UTC] USER=www-data EUID=0 PID=1553116 ACTION=fsop ARGS=cp /tmp/tmp.1VKeUCfunA /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[INFO] Allocated new IP for telemetry: 10.100.1.210
[2026-01-02 06:38:27 UTC] USER=www-data EUID=0 PID=1553133 ACTION=fsop ARGS=cp /tmp/tmp.LZe7MlWF9B /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[INFO] Allocated observability IPs:
[INFO] metrics: 10.100.1.205
[INFO] alerts: 10.100.1.209
[INFO] dashboards: 10.100.1.206
[INFO] traces: 10.100.1.208
[INFO] telemetry: 10.100.1.210
[INFO] logstore: 10.100.1.207
[INFO] proxy: 10.100.1.51
[INFO] obs: 10.100.1.51
[ OK ] Network infrastructure allocated
[INFO] Cleaning up ports from previous environments...
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking and cleaning ports for observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m IP Address: 10.100.1.51
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking for conflicting observability services...
[0;34m[INFO][0m Service clickhouse-server-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service clickhouse-server-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Found 14 observability service(s) (all belong to current cell)
[0;34m[INFO][0m Checking for remaining processes on IP 10.100.1.51...
[0;34m[INFO][0m Scanning 15 ports...
[0;32m[OK][0m ✅ All 15 ports are FREE - ready for installation
[0;32m[OK][0m Port cleanup completed successfully
[0;34m[INFO][0m Configuring IP aliases on network interface...
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m CONFIGURING NETWORK IP ALIASES
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Interface: lo
[0;34m[INFO][0m IP Count: 8
[0;34m[INFO][0m Configuring: metrics → 10.100.1.205
[0;34m[INFO][0m Configuring IP alias: 10.100.1.205/32 on lo
[0;32m[OK][0m ✅ IP 10.100.1.205 configured successfully on lo
[0;32m[OK][0m ✅ IP 10.100.1.205 verified on network interface
[0;34m[INFO][0m Configuring: alerts → 10.100.1.209
[0;34m[INFO][0m Configuring IP alias: 10.100.1.209/32 on lo
[0;32m[OK][0m ✅ IP 10.100.1.209 configured successfully on lo
[0;32m[OK][0m ✅ IP 10.100.1.209 verified on network interface
[0;34m[INFO][0m Configuring: dashboards → 10.100.1.206
[0;34m[INFO][0m Configuring IP alias: 10.100.1.206/32 on lo
[0;32m[OK][0m ✅ IP 10.100.1.206 configured successfully on lo
[0;32m[OK][0m ✅ IP 10.100.1.206 verified on network interface
[0;34m[INFO][0m Configuring: traces → 10.100.1.208
[0;34m[INFO][0m Configuring IP alias: 10.100.1.208/32 on lo
[0;32m[OK][0m ✅ IP 10.100.1.208 configured successfully on lo
[0;32m[OK][0m ✅ IP 10.100.1.208 verified on network interface
[0;34m[INFO][0m Configuring: telemetry → 10.100.1.210
[0;34m[INFO][0m Configuring IP alias: 10.100.1.210/32 on lo
[0;32m[OK][0m ✅ IP 10.100.1.210 configured successfully on lo
[0;32m[OK][0m ✅ IP 10.100.1.210 verified on network interface
[0;34m[INFO][0m Configuring: logstore → 10.100.1.207
[0;34m[INFO][0m Configuring IP alias: 10.100.1.207/32 on lo
[0;32m[OK][0m ✅ IP 10.100.1.207 configured successfully on lo
[0;32m[OK][0m ✅ IP 10.100.1.207 verified on network interface
[0;34m[INFO][0m Configuring: proxy → 10.100.1.51
[0;34m[INFO][0m Configuring IP alias: 10.100.1.51/32 on lo
[0;32m[OK][0m ✅ IP 10.100.1.51 configured successfully on lo
[0;32m[OK][0m ✅ IP 10.100.1.51 verified on network interface
[0;34m[INFO][0m Configuring: obs → 10.100.1.51
[0;34m[INFO][0m IP 10.100.1.51 already configured on network interface
[0;32m[OK][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ All IP aliases configured successfully
[0;32m[OK][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Current IP configuration on lo:
inet 127.0.0.1/8 scope host lo
inet 10.100.60.2/32 scope global lo
inet 10.100.1.182/32 scope global lo
inet 10.100.1.187/32 scope global lo
inet 10.100.1.183/32 scope global lo
inet 10.100.1.186/32 scope global lo
inet 10.100.1.188/32 scope global lo
inet 10.100.1.184/32 scope global lo
inet 10.100.1.181/32 scope global lo
inet 10.100.1.192/32 scope global lo:pgbouncer
inet 10.100.1.193/32 scope global lo
inet 10.100.1.197/32 scope global lo
inet 10.100.1.194/32 scope global lo
inet 10.100.1.196/32 scope global lo
inet 10.100.1.198/32 scope global lo
inet 10.100.1.195/32 scope global lo
inet 10.100.1.180/32 scope global lo
inet 10.100.1.179/32 scope global lo
inet 10.100.1.205/32 scope global lo
inet 10.100.1.209/32 scope global lo
inet 10.100.1.206/32 scope global lo
inet 10.100.1.208/32 scope global lo
inet 10.100.1.210/32 scope global lo
inet 10.100.1.207/32 scope global lo
inet 10.100.1.51/32 scope global lo
[0;32m[OK][0m IP aliases configured on network interface
[0;34m[INFO][0m Step 2/10: Creating DNS entries...
[0;34m[INFO][0m Configuring DNS entries in /etc/hosts...
[0;34m[INFO][0m Added: metrics-identity-sau-main-dev-prometheus.fastorder.com → 10.100.1.205
[0;34m[INFO][0m Added: alerts-identity-sau-main-dev-alertmanager.fastorder.com → 10.100.1.209
[0;34m[INFO][0m Added: dashboards-identity-sau-main-dev-grafana.fastorder.com → 10.100.1.206
[0;34m[INFO][0m Added: traces-identity-sau-main-dev-tempo.fastorder.com → 10.100.1.208
[0;34m[INFO][0m Added: telemetry-identity-sau-main-dev-opentelemetry.fastorder.com → 10.100.1.210
[0;34m[INFO][0m Added: logstore-identity-sau-main-dev-clickhouse.fastorder.com → 10.100.1.207
[0;34m[INFO][0m Added: observe-identity-sau-main-dev.fastorder.com → 10.100.1.51
[0;34m[INFO][0m Adding observability integration aliases...
[0;34m[INFO][0m Added alias: metrics-identity-sau-main-dev.fastorder.com → 10.100.1.205
[0;34m[INFO][0m Added alias: alerts-identity-sau-main-dev.fastorder.com → 10.100.1.209
[0;34m[INFO][0m Added alias: dashboards-identity-sau-main-dev.fastorder.com → 10.100.1.206
[0;34m[INFO][0m Added alias: traces-identity-sau-main-dev.fastorder.com → 10.100.1.208
[0;34m[INFO][0m Added alias: telemetry-identity-sau-main-dev.fastorder.com → 10.100.1.210
[0;34m[INFO][0m Added alias: logstore-identity-sau-main-dev.fastorder.com → 10.100.1.207
[2026-01-02 06:38:28 UTC] USER=www-data EUID=0 PID=1553557 ACTION=fsop ARGS=sed -i /observe-identity-sau-main-dev.fastorder.com/d /etc/hosts
[0;34m[INFO][0m Added alias: observe-identity-sau-main-dev.fastorder.com → 10.100.1.51
[0;32m[OK][0m DNS entries created
[0;34m[INFO][0m Step 3/10: Creating AWS Secrets Manager structure...
[INFO] Creating AWS Secrets Manager structure
[INFO] Base path: fastorder/observability/identity/sau/dev
[INFO] Observability Cell: obs-identity-sau-main-dev
[INFO] Application Cell: identity-sau-main-dev
[INFO] Exists: fastorder/observability/identity/sau/dev/metrics
[INFO] Exists: fastorder/observability/identity/sau/dev/dashboards
[INFO] Exists: fastorder/observability/identity/sau/dev/logstore
[INFO] Exists: fastorder/observability/identity/sau/dev/traces
[INFO] Exists: fastorder/observability/identity/sau/dev/telemetry
[INFO] Exists: fastorder/observability/identity/sau/dev/alerts
[INFO] Secrets structure created successfully
[0;32m[OK][0m Secrets structure created
[0;34m[INFO][0m Step 4/10: Generating mTLS certificates...
[INFO] Generating mTLS certificates for observability cell
[INFO] Observability Cell: obs-identity-sau-main-dev
[INFO] Components: prometheus,grafana,loki,tempo,otlp_collector,clickhouse,alertmanager
[INFO] Creating certificate directory: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[2026-01-02 06:38:43 UTC] USER=www-data EUID=0 PID=1553723 ACTION=fsop ARGS=mkdir -p /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[2026-01-02 06:38:43 UTC] USER=www-data EUID=0 PID=1553732 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[INFO] Generating CA certificate for obs-identity-sau-main-dev
[2026-01-02 06:38:43 UTC] USER=www-data EUID=0 PID=1553741 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem 4096
[2026-01-02 06:38:45 UTC] USER=www-data EUID=0 PID=1553780 ACTION=fsop ARGS=openssl req -new -x509 -days 3650 -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=Observability/CN=obs-identity-sau-main-dev-ca
[2026-01-02 06:38:45 UTC] USER=www-data EUID=0 PID=1553826 ACTION=fsop ARGS=chmod 600 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem
[2026-01-02 06:38:45 UTC] USER=www-data EUID=0 PID=1553849 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[INFO] CA certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[INFO] Generating certificate for: prometheus
[2026-01-02 06:38:45 UTC] USER=www-data EUID=0 PID=1553881 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-key.pem 2048
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553911 ACTION=fsop ARGS=openssl req -new -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-csr.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=Observability/CN=prometheus.obs-identity-sau-main-dev
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553921 ACTION=fsop ARGS=openssl x509 -req -in /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-csr.pem -CA /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -CAkey /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -CAcreateserial -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-cert.pem -days 730
Certificate request self-signature ok
subject=C = US, ST = State, L = City, O = FastOrder, OU = Observability, CN = prometheus.obs-identity-sau-main-dev
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553930 ACTION=fsop ARGS=chmod 600 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-key.pem
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553939 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-cert.pem
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553948 ACTION=fsop ARGS=rm -f /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-csr.pem
[INFO] Certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-cert.pem
[INFO] Generating certificate for: grafana
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553957 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/grafana-key.pem 2048
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553966 ACTION=fsop ARGS=openssl req -new -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/grafana-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/grafana-csr.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=Observability/CN=grafana.obs-identity-sau-main-dev
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553977 ACTION=fsop ARGS=openssl x509 -req -in /etc/fastorder/observability/certs/obs-identity-sau-main-dev/grafana-csr.pem -CA /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -CAkey /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -CAcreateserial -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/grafana-cert.pem -days 730
Certificate request self-signature ok
subject=C = US, ST = State, L = City, O = FastOrder, OU = Observability, CN = grafana.obs-identity-sau-main-dev
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553986 ACTION=fsop ARGS=chmod 600 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/grafana-key.pem
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1553995 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/grafana-cert.pem
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1554004 ACTION=fsop ARGS=rm -f /etc/fastorder/observability/certs/obs-identity-sau-main-dev/grafana-csr.pem
[INFO] Certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/grafana-cert.pem
[INFO] Generating certificate for: loki
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1554013 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/loki-key.pem 2048
[2026-01-02 06:38:46 UTC] USER=www-data EUID=0 PID=1554022 ACTION=fsop ARGS=openssl req -new -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/loki-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/loki-csr.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=Observability/CN=loki.obs-identity-sau-main-dev
[2026-01-02 06:38:47 UTC] USER=www-data EUID=0 PID=1554031 ACTION=fsop ARGS=openssl x509 -req -in /etc/fastorder/observability/certs/obs-identity-sau-main-dev/loki-csr.pem -CA /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -CAkey /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -CAcreateserial -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/loki-cert.pem -days 730
Certificate request self-signature ok
subject=C = US, ST = State, L = City, O = FastOrder, OU = Observability, CN = loki.obs-identity-sau-main-dev
[2026-01-02 06:38:47 UTC] USER=www-data EUID=0 PID=1554040 ACTION=fsop ARGS=chmod 600 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/loki-key.pem
[2026-01-02 06:38:47 UTC] USER=www-data EUID=0 PID=1554049 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/loki-cert.pem
[INFO] Certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/loki-cert.pem
[INFO] Generating certificate for: tempo
[2026-01-02 06:38:47 UTC] USER=www-data EUID=0 PID=1554072 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-key.pem 2048
[2026-01-02 06:38:47 UTC] USER=www-data EUID=0 PID=1554081 ACTION=fsop ARGS=openssl req -new -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-csr.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=Observability/CN=tempo.obs-identity-sau-main-dev
[2026-01-02 06:38:47 UTC] USER=www-data EUID=0 PID=1554095 ACTION=fsop ARGS=openssl x509 -req -in /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-csr.pem -CA /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -CAkey /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -CAcreateserial -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-cert.pem -days 730
Certificate request self-signature ok
subject=C = US, ST = State, L = City, O = FastOrder, OU = Observability, CN = tempo.obs-identity-sau-main-dev
[2026-01-02 06:38:47 UTC] USER=www-data EUID=0 PID=1554104 ACTION=fsop ARGS=chmod 600 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-key.pem
[2026-01-02 06:38:47 UTC] USER=www-data EUID=0 PID=1554113 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-cert.pem
[2026-01-02 06:38:48 UTC] USER=www-data EUID=0 PID=1554122 ACTION=fsop ARGS=rm -f /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-csr.pem
[INFO] Certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-cert.pem
[INFO] Generating certificate for: otlp_collector
[2026-01-02 06:38:48 UTC] USER=www-data EUID=0 PID=1554131 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-key.pem 2048
[2026-01-02 06:38:48 UTC] USER=www-data EUID=0 PID=1554143 ACTION=fsop ARGS=openssl req -new -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-csr.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=Observability/CN=otlp_collector.obs-identity-sau-main-dev
[2026-01-02 06:38:48 UTC] USER=www-data EUID=0 PID=1554152 ACTION=fsop ARGS=openssl x509 -req -in /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-csr.pem -CA /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -CAkey /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -CAcreateserial -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-cert.pem -days 730
Certificate request self-signature ok
subject=C = US, ST = State, L = City, O = FastOrder, OU = Observability, CN = otlp_collector.obs-identity-sau-main-dev
[2026-01-02 06:38:48 UTC] USER=www-data EUID=0 PID=1554161 ACTION=fsop ARGS=chmod 600 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-key.pem
[2026-01-02 06:38:48 UTC] USER=www-data EUID=0 PID=1554170 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-cert.pem
[2026-01-02 06:38:48 UTC] USER=www-data EUID=0 PID=1554179 ACTION=fsop ARGS=rm -f /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-csr.pem
[INFO] Certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-cert.pem
[INFO] Generating certificate for: clickhouse
[2026-01-02 06:38:48 UTC] USER=www-data EUID=0 PID=1554188 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-key.pem 2048
[2026-01-02 06:38:49 UTC] USER=www-data EUID=0 PID=1554203 ACTION=fsop ARGS=openssl req -new -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-csr.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=Observability/CN=clickhouse.obs-identity-sau-main-dev
[2026-01-02 06:38:49 UTC] USER=www-data EUID=0 PID=1554214 ACTION=fsop ARGS=openssl x509 -req -in /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-csr.pem -CA /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -CAkey /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -CAcreateserial -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-cert.pem -days 730
Certificate request self-signature ok
subject=C = US, ST = State, L = City, O = FastOrder, OU = Observability, CN = clickhouse.obs-identity-sau-main-dev
[2026-01-02 06:38:49 UTC] USER=www-data EUID=0 PID=1554223 ACTION=fsop ARGS=chmod 600 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-key.pem
[2026-01-02 06:38:49 UTC] USER=www-data EUID=0 PID=1554232 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-cert.pem
[2026-01-02 06:38:49 UTC] USER=www-data EUID=0 PID=1554241 ACTION=fsop ARGS=rm -f /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-csr.pem
[INFO] Certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-cert.pem
[INFO] Generating certificate for: alertmanager
[2026-01-02 06:38:49 UTC] USER=www-data EUID=0 PID=1554251 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-key.pem 2048
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554264 ACTION=fsop ARGS=openssl req -new -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-csr.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=Observability/CN=alertmanager.obs-identity-sau-main-dev
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554273 ACTION=fsop ARGS=openssl x509 -req -in /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-csr.pem -CA /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -CAkey /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -CAcreateserial -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-cert.pem -days 730
Certificate request self-signature ok
subject=C = US, ST = State, L = City, O = FastOrder, OU = Observability, CN = alertmanager.obs-identity-sau-main-dev
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554282 ACTION=fsop ARGS=chmod 600 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-key.pem
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554291 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-cert.pem
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554300 ACTION=fsop ARGS=rm -f /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-csr.pem
[INFO] Certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-cert.pem
[INFO] Generating PHP client certificate for metrics service...
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554309 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-key.pem 2048
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554318 ACTION=fsop ARGS=openssl req -new -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-csr.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=Dashboard/CN=php-metrics-client.obs-identity-sau-main-dev
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554327 ACTION=fsop ARGS=openssl x509 -req -in /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-csr.pem -CA /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -CAkey /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -CAcreateserial -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-cert.pem -days 730
Certificate request self-signature ok
subject=C = US, ST = State, L = City, O = FastOrder, OU = Dashboard, CN = php-metrics-client.obs-identity-sau-main-dev
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554336 ACTION=fsop ARGS=chmod 640 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-key.pem
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554345 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-cert.pem
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554354 ACTION=fsop ARGS=chown root:www-data /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-key.pem
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554363 ACTION=fsop ARGS=chown root:www-data /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-cert.pem
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554372 ACTION=fsop ARGS=rm -f /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-csr.pem
[INFO] PHP client certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-cert.pem
[INFO] Generating Apache client certificate for mTLS reverse proxy...
[2026-01-02 06:38:50 UTC] USER=www-data EUID=0 PID=1554381 ACTION=fsop ARGS=openssl genrsa -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-key.pem 2048
[2026-01-02 06:38:51 UTC] USER=www-data EUID=0 PID=1554398 ACTION=fsop ARGS=openssl req -new -key /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-key.pem -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-csr.pem -subj /C=US/ST=State/L=City/O=FastOrder/OU=ReverseProxy/CN=apache-proxy.obs-identity-sau-main-dev
[2026-01-02 06:38:51 UTC] USER=www-data EUID=0 PID=1554409 ACTION=fsop ARGS=openssl x509 -req -in /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-csr.pem -CA /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem -CAkey /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-key.pem -CAcreateserial -out /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-cert.pem -days 730
Certificate request self-signature ok
subject=C = US, ST = State, L = City, O = FastOrder, OU = ReverseProxy, CN = apache-proxy.obs-identity-sau-main-dev
[2026-01-02 06:38:51 UTC] USER=www-data EUID=0 PID=1554437 ACTION=fsop ARGS=chmod 640 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-key.pem
[2026-01-02 06:38:51 UTC] USER=www-data EUID=0 PID=1554450 ACTION=fsop ARGS=chmod 640 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-combined.pem
[2026-01-02 06:38:51 UTC] USER=www-data EUID=0 PID=1554459 ACTION=fsop ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-cert.pem
[2026-01-02 06:38:51 UTC] USER=www-data EUID=0 PID=1554468 ACTION=fsop ARGS=chown root:www-data /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-key.pem
[2026-01-02 06:38:51 UTC] USER=www-data EUID=0 PID=1554477 ACTION=fsop ARGS=chown root:www-data /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-cert.pem
[2026-01-02 06:38:51 UTC] USER=www-data EUID=0 PID=1554487 ACTION=fsop ARGS=chown root:www-data /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-combined.pem
[2026-01-02 06:38:52 UTC] USER=www-data EUID=0 PID=1554496 ACTION=fsop ARGS=rm -f /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-csr.pem
[INFO] Apache client certificate created: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-cert.pem
[INFO] Apache combined cert+key: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-combined.pem
[INFO] Storing mTLS certificates in AWS Secrets Manager...
{
"ARN": "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/observability/identity/sau/main/dev/mtls/php-client-qgBJOa",
"Name": "fastorder/observability/identity/sau/main/dev/mtls/php-client",
"VersionId": "5df2df92-7298-46dc-aa3e-169a22cbb095"
}
[INFO] mTLS certificates stored in Secrets Manager: fastorder/observability/identity/sau/main/dev/mtls/php-client
[INFO] mTLS certificates generated successfully
[INFO] Certificate directory: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[INFO] PHP client cert: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-cert.pem
[INFO] PHP client key: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/php-client-key.pem
[INFO] Apache client cert: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-cert.pem
[INFO] Apache combined (for SSLProxyMachineCertificateFile): /etc/fastorder/observability/certs/obs-identity-sau-main-dev/apache-client-combined.pem
[0;32m[OK][0m mTLS certificates generated
[0;34m[INFO][0m Step 5/10: Deploying log storage backend...
[0;34m[INFO][0m Provider: clickhouse (selected)
[0;34m[INFO][0m Note: Deployed before telemetry (OtelCol depends on log storage)
[0;34m[INFO][0m FQDN: logstore-identity-sau-main-dev-clickhouse.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.207
[0;34m[INFO][0m Deploying log backend: clickhouse...
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m LOG STORAGE BACKEND DEPLOYMENT
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: clickhouse
[0;34m[INFO][0m Observability Cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: logstore-identity-sau-main-dev-clickhouse.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.207
[0;34m[INFO][0m S3 Bucket: fastorder-logs-sau-dev
[0;34m[INFO][0m Retention: 90 days
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[2026-01-02 06:38:54 UTC] USER=unknown EUID=33 PID=1554541 ACTION=fsop ARGS=chmod +x /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/LogStorageBackend/provider/clickhouse.sh
/bin/chmod: changing permissions of '/opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/LogStorageBackend/provider/clickhouse.sh': Operation not permitted
[0;34m[INFO][0m Using provider: clickhouse
[0;34m[INFO][0m Provider script: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/LogStorageBackend/provider/clickhouse.sh
[0;34m[INFO][0m Executing provider deployment script...
[0;34m[INFO][0m Parsed: SERVICE=identity, ZONE=sau, BRANCH=main, ENV=dev
[0;34m[INFO][0m Checking and cleaning ports before installation...
[0;34m[INFO][0m Initializing certificate directory for obs-identity-sau-main-dev...
[2026-01-02 06:38:54 UTC] USER=www-data EUID=0 PID=1554558 ACTION=passthru ARGS=chmod 755 /etc/fastorder
[2026-01-02 06:38:54 UTC] USER=www-data EUID=0 PID=1554567 ACTION=passthru ARGS=chmod 755 /etc/fastorder/observability
[2026-01-02 06:38:54 UTC] USER=www-data EUID=0 PID=1554576 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs
[2026-01-02 06:38:54 UTC] USER=www-data EUID=0 PID=1554585 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;32m[OK][0m Certificate directory initialized: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking and cleaning ports for observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m IP Address: 10.100.1.207
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking for conflicting observability services...
[0;34m[INFO][0m Service clickhouse-server-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service clickhouse-server-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Found 14 observability service(s) (all belong to current cell)
[0;34m[INFO][0m Checking for remaining processes on IP 10.100.1.207...
[0;34m[INFO][0m Scanning 15 ports...
[0;32m[OK][0m ✅ All 15 ports are FREE - ready for installation
[0;32m[OK][0m Port cleanup successful on attempt 1
[0;34m[INFO][0m Binding ClickHouse to allocated IP: 10.100.1.207
[0;34m[INFO][0m Deploying ClickHouse for obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: logstore-identity-sau-main-dev-clickhouse.fastorder.com
[0;34m[INFO][0m Allocated IP: 10.100.1.207
[0;34m[INFO][0m VM IP: 10.100.1.207
[0;34m[INFO][0m Ports: HTTP=8123 TCP=9000 Interserver=9009
[0;34m[INFO][0m S3 Bucket: fastorder-logs-sau-dev (region=me-central-1)
[0;34m[INFO][0m Retention: 90 days
[0;34m[INFO][0m Checking if ClickHouse is installed...
[0;32m[OK][0m ClickHouse already installed
[2026-01-02 06:38:55 UTC] USER=www-data EUID=0 PID=1554757 ACTION=fsop ARGS=mkdir -p /etc/clickhouse-server-obs-identity-sau-main-dev/config.d
[2026-01-02 06:38:55 UTC] USER=www-data EUID=0 PID=1554766 ACTION=fsop ARGS=mkdir -p /etc/clickhouse-server-obs-identity-sau-main-dev/users.d
[2026-01-02 06:38:55 UTC] USER=www-data EUID=0 PID=1554790 ACTION=fsop ARGS=mkdir -p /var/log/clickhouse-server-obs-identity-sau-main-dev
[2026-01-02 06:38:55 UTC] USER=www-data EUID=0 PID=1554799 ACTION=passthru ARGS=chmod 755 /etc/clickhouse-server-obs-identity-sau-main-dev
[2026-01-02 06:38:55 UTC] USER=www-data EUID=0 PID=1554808 ACTION=passthru ARGS=chmod 700 /var/lib/clickhouse-obs-identity-sau-main-dev
[2026-01-02 06:38:55 UTC] USER=www-data EUID=0 PID=1554817 ACTION=passthru ARGS=chmod 750 /var/log/clickhouse-server-obs-identity-sau-main-dev
[0;34m[INFO][0m No existing logs_writer credentials found - generating new ones
[0;34m[INFO][0m No existing metrics_reader credentials found - generating new ones
[0;34m[INFO][0m TLS configuration exported for clickhouse
[0;34m[INFO][0m Cert: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-cert.pem
[0;34m[INFO][0m Key: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-key.pem
[0;34m[INFO][0m CA: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[0;34m[INFO][0m Configuring certificate permissions for clickhouse (user: clickhouse)
[0;34m[INFO][0m Initializing certificate directory for obs-identity-sau-main-dev...
[2026-01-02 06:39:00 UTC] USER=www-data EUID=0 PID=1554890 ACTION=passthru ARGS=chmod 755 /etc/fastorder
[2026-01-02 06:39:00 UTC] USER=www-data EUID=0 PID=1554899 ACTION=passthru ARGS=chmod 755 /etc/fastorder/observability
[2026-01-02 06:39:00 UTC] USER=www-data EUID=0 PID=1554908 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs
[2026-01-02 06:39:00 UTC] USER=www-data EUID=0 PID=1554917 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;32m[OK][0m Certificate directory initialized: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;34m[INFO][0m Setting file permissions...
[2026-01-02 06:39:00 UTC] USER=www-data EUID=0 PID=1554937 ACTION=passthru ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-cert.pem
[2026-01-02 06:39:00 UTC] USER=www-data EUID=0 PID=1554946 ACTION=passthru ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[2026-01-02 06:39:00 UTC] USER=www-data EUID=0 PID=1554968 ACTION=passthru ARGS=chmod 640 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-key.pem
[0;34m[INFO][0m Setting file ownership...
[2026-01-02 06:39:00 UTC] USER=www-data EUID=0 PID=1554979 ACTION=passthru ARGS=chown root:clickhouse /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-key.pem
[2026-01-02 06:39:00 UTC] USER=www-data EUID=0 PID=1554988 ACTION=passthru ARGS=chown root:root /etc/fastorder/observability/certs/obs-identity-sau-main-dev/clickhouse-cert.pem /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[0;34m[INFO][0m Permission configuration completed
[0;34m[INFO][0m (Verification skipped - running via wrapper, trust chmod/chown success)
[0;32m[OK][0m ✅ Certificate permissions configured successfully for clickhouse
[0;34m[INFO][0m Creating ClickHouse configuration...
[2026-01-02 06:39:01 UTC] USER=www-data EUID=0 PID=1555060 ACTION=passthru ARGS=chown -R clickhouse:clickhouse /etc/clickhouse-server-obs-identity-sau-main-dev
[2026-01-02 06:39:01 UTC] USER=www-data EUID=0 PID=1555069 ACTION=passthru ARGS=bash -c chmod 640 /etc/clickhouse-server-obs-identity-sau-main-dev/*.xml
[0;32m[OK][0m ClickHouse configuration created
[0;34m[INFO][0m Creating logs table schema...
[2026-01-02 06:39:01 UTC] USER=www-data EUID=0 PID=1555102 ACTION=passthru ARGS=sed -i s/__RETENTION_DAYS__/90/g /etc/clickhouse-server-obs-identity-sau-main-dev/logs_schema.sql
[2026-01-02 06:39:01 UTC] USER=www-data EUID=0 PID=1555111 ACTION=passthru ARGS=chmod 644 /etc/clickhouse-server-obs-identity-sau-main-dev/logs_schema.sql
[0;32m[OK][0m Logs schema created
[0;34m[INFO][0m Creating systemd service...
[2026-01-02 06:39:01 UTC] USER=www-data EUID=0 PID=1555142 ACTION=passthru ARGS=chown -R clickhouse:clickhouse /var/lib/clickhouse-obs-identity-sau-main-dev
[2026-01-02 06:39:01 UTC] USER=www-data EUID=0 PID=1555151 ACTION=passthru ARGS=chown -R clickhouse:clickhouse /var/log/clickhouse-server-obs-identity-sau-main-dev
[2026-01-02 06:39:01 UTC] USER=www-data EUID=0 PID=1555173 ACTION=passthru ARGS=chmod 700 /var/lib/clickhouse-obs-identity-sau-main-dev
[0;32m[OK][0m Systemd service created
[0;34m[INFO][0m Starting ClickHouse service...
[2026-01-02 06:39:01 UTC] USER=www-data EUID=0 PID=1555184 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:39:02 UTC] USER=www-data EUID=0 PID=1555300 ACTION=passthru ARGS=systemctl enable clickhouse-server-obs-identity-sau-main-dev.service
Created symlink /etc/systemd/system/multi-user.target.wants/clickhouse-server-obs-identity-sau-main-dev.service -> /etc/systemd/system/clickhouse-server-obs-identity-sau-main-dev.service.
[2026-01-02 06:39:03 UTC] USER=www-data EUID=0 PID=1555349 ACTION=passthru ARGS=systemctl start clickhouse-server-obs-identity-sau-main-dev.service
[0;34m[INFO][0m Waiting for ClickHouse to be ready...
[0;32m[OK][0m ClickHouse is ready
[0;34m[INFO][0m Initializing database schema...
[0;32m[OK][0m Schema initialized
[0;34m[INFO][0m Storing ClickHouse credentials in AWS Secrets Manager...
{
"ARN": "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/observability/identity/sau/main/dev/clickhouse/server/logs_writer-G4ex4t",
"Name": "fastorder/observability/identity/sau/main/dev/clickhouse/server/logs_writer",
"VersionId": "0b36aa94-baba-43f2-9283-95f3055996b3"
}
[0;32m[OK][0m logs_writer credentials stored and verified in Secrets Manager
{
"ARN": "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/observability/identity/sau/main/dev/clickhouse/server/metrics_reader-zM91Q1",
"Name": "fastorder/observability/identity/sau/main/dev/clickhouse/server/metrics_reader",
"VersionId": "078999bb-b2f0-4bb1-8803-4dff702fb3b8"
}
[0;32m[OK][0m metrics_reader credentials stored and verified in Secrets Manager
[0;34m[INFO][0m Validating ClickHouse deployment...
[0;34m[INFO][0m ClickHouse version: 25.10.1.3832
[0;34m[INFO][0m Tables created: .inner_id.c4ac0936-c248-4ceb-840e-5e9ef70252d3
.inner_id.f8978431-11f3-47cb-a843-4a7f362d3e16
application_logs
error_logs_mv
iam_audit_event
metrics_all
otel_logs
request_logs_mv
security_access
[0;34m[INFO][0m Test log inserted. Total logs: 1
[0;32m[OK][0m ✅ ClickHouse deployment validated
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ ClickHouse Deployed Successfully
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m FQDN: logstore-identity-sau-main-dev-clickhouse.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.207
[0;34m[INFO][0m HTTP Port: 8123
[0;34m[INFO][0m Native Port: 9000
[0;34m[INFO][0m Database: logs
[0;34m[INFO][0m Retention: 90 days
[0;34m[INFO][0m Storage: Tiered (Local → S3: fastorder-logs-sau-dev in me-central-1)
[0;34m[INFO][0m
[0;34m[INFO][0m Credentials stored in AWS Secrets Manager:
[0;34m[INFO][0m Writers: fastorder/observability/identity/sau/main/dev/clickhouse/server/logs_writer
[0;34m[INFO][0m Readers: fastorder/observability/identity/sau/main/dev/clickhouse/server/metrics_reader (for PHP metrics service)
[0;34m[INFO][0m
[0;34m[INFO][0m Example queries (using credentials from Secrets Manager):
[0;34m[INFO][0m # Write logs:
[0;34m[INFO][0m clickhouse-client --host logstore-identity-sau-main-dev-clickhouse.fastorder.com --port 9000 --user logs_writer --password '***' --query 'SELECT 1'
[0;34m[INFO][0m
[0;34m[INFO][0m # Read metrics (PHP metrics service):
[0;34m[INFO][0m clickhouse-client --host logstore-identity-sau-main-dev-clickhouse.fastorder.com --port 9000 --user metrics_reader --password '***' --query 'SELECT * FROM system.metrics'
[0;34m[INFO][0m
[0;34m[INFO][0m HTTPS Setup (run on web-03/skeleton server):
[0;34m[INFO][0m # Set up HTTPS reverse proxy with Let's Encrypt:
[0;34m[INFO][0m OBS_CELL=obs-identity-sau-main-dev BACKEND_IP=10.100.1.207 sudo bash /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/LogStorageBackend/provider/../https/setup-clickhouse-https.sh
[0;34m[INFO][0m
[0;34m[INFO][0m # Or add --setup-https flag when running this script
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ Log Storage Backend Deployed Successfully
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: clickhouse
[0;34m[INFO][0m FQDN: logstore-identity-sau-main-dev-clickhouse.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.207
[0;34m[INFO][0m Retention: 90 days
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Registering ClickHouse in monitoring dashboard...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: ClickHouse
[INFO] Identifier: identity-sau-main-dev-clickhouse
[INFO] Identifier Parent: cluster
[INFO] IP: 10.100.1.207
[INFO] Port: 8443
[INFO] FQDN: logstore-identity-sau-main-dev-clickhouse.fastorder.com
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[ERROR] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[ERROR] ❌ INVALID REQUEST
[ERROR] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[ERROR] Response: {"success":false,"error":"Invalid JSON: Control character error, possibly incorrectly encoded"}
[ERROR]
[ERROR] Request payload:
{
"env_id": "identity-sau-main-dev",
"application": "ClickHouse",
"identifier": "identity-sau-main-dev-clickhouse",
"identifier_parent": "cluster",
"ip": "10.100.1.207",
"port": 8443,
"fqdn": "logstore-identity-sau-main-dev-clickhouse.fastorder.com",
"status": "running",
"meta": {
"role": "log_storage",
"provider": "clickhouse",
"version": "25.10
1.3832",
"http_port": 8123,
"native_port": 9000,
"https_port": 8443,
"protocol": "https",
"metrics_enabled": true,
"metrics_port": 8123,
"metrics_path": "/metrics",
"health_endpoint": "https://logstore-identity-sau-main-dev-clickhouse.fastorder.com/ping",
"retention_days": 90,
"s3_bucket": "fastorder-logs-sau-dev"
}
}
[ERROR] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[1;33m[WARN][0m ⚠️ Failed to register ClickHouse (service is running)
[0;32m[OK][0m clickhouse deployed successfully
[0;32m[OK][0m Log storage backend deployed
[0;34m[INFO][0m Step 6/10: Deploying telemetry collector...
[0;34m[INFO][0m Provider: otlp (backend implementation - internal)
[0;34m[INFO][0m Endpoint: telemetry-identity-sau-main-dev-opentelemetry.fastorder.com (stable, exposed to clients)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m TELEMETRY COLLECTOR DEPLOYMENT
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: otlp
[0;34m[INFO][0m Observability Cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: telemetry-identity-sau-main-dev-opentelemetry.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.210
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Using provider: otlp
[0;34m[INFO][0m Provider script: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Telemetry/provider/otlp.sh
[0;34m[INFO][0m Executing provider deployment script...
[0;34m[INFO][0m Parsed: SERVICE=identity, ZONE=sau, BRANCH=main, ENV=dev
[0;34m[INFO][0m Checking and cleaning ports before installation...
[0;34m[INFO][0m Initializing certificate directory for obs-identity-sau-main-dev...
[2026-01-02 06:39:21 UTC] USER=www-data EUID=0 PID=1556392 ACTION=passthru ARGS=chmod 755 /etc/fastorder
[2026-01-02 06:39:21 UTC] USER=www-data EUID=0 PID=1556401 ACTION=passthru ARGS=chmod 755 /etc/fastorder/observability
[2026-01-02 06:39:21 UTC] USER=www-data EUID=0 PID=1556410 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs
[2026-01-02 06:39:21 UTC] USER=www-data EUID=0 PID=1556419 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;32m[OK][0m Certificate directory initialized: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking and cleaning ports for observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m IP Address: 10.100.1.210
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking for conflicting observability services...
[0;34m[INFO][0m Service clickhouse-server-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service clickhouse-server-obs-identity-sau-main-dev.service belongs to current cell (skipping)
[0;34m[INFO][0m Service clickhouse-server-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Found 15 observability service(s) (all belong to current cell)
[0;34m[INFO][0m Checking for remaining processes on IP 10.100.1.210...
[0;34m[INFO][0m Scanning 15 ports...
[0;32m[OK][0m ✅ All 15 ports are FREE - ready for installation
[0;32m[OK][0m Port cleanup successful on attempt 1
[0;34m[INFO][0m Binding to allocated IP: 10.100.1.210
[0;34m[INFO][0m Deploying OpenTelemetry Collector for observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: telemetry-identity-sau-main-dev-opentelemetry.fastorder.com
[0;34m[INFO][0m Allocated IP: 10.100.1.210
[0;34m[INFO][0m VM IP: 10.100.1.210
[0;34m[INFO][0m Ports: gRPC=4317 HTTP=4318 Metrics=8888 Prom=8889
[0;32m[OK][0m User 'otelcol' already exists
[0;34m[INFO][0m Checking if OpenTelemetry Collector is installed...
[0;32m[OK][0m OpenTelemetry Collector already installed at /usr/local/bin/otelcol-contrib
[0;34m[INFO][0m Creating configuration/data directories...
[2026-01-02 06:39:22 UTC] USER=www-data EUID=0 PID=1556597 ACTION=passthru ARGS=mkdir -p /etc/otelcol/obs-identity-sau-main-dev
[2026-01-02 06:39:22 UTC] USER=www-data EUID=0 PID=1556606 ACTION=passthru ARGS=mkdir -p /var/lib/otelcol/obs-identity-sau-main-dev
[2026-01-02 06:39:22 UTC] USER=www-data EUID=0 PID=1556618 ACTION=passthru ARGS=chown -R otelcol:otelcol /etc/otelcol/obs-identity-sau-main-dev /var/lib/otelcol/obs-identity-sau-main-dev
[2026-01-02 06:39:22 UTC] USER=www-data EUID=0 PID=1556629 ACTION=passthru ARGS=chmod 0750 /etc/otelcol/obs-identity-sau-main-dev
[2026-01-02 06:39:22 UTC] USER=www-data EUID=0 PID=1556638 ACTION=passthru ARGS=chmod 0750 /var/lib/otelcol/obs-identity-sau-main-dev
[0;34m[INFO][0m Retrieving ClickHouse credentials from Secrets Manager...
[0;32m[OK][0m Retrieved ClickHouse credentials from Secrets Manager
[0;34m[INFO][0m Creating OpenTelemetry Collector configuration...
[0;34m[INFO][0m ClickHouse exporter enabled: tcp://logstore-identity-sau-main-dev-clickhouse.fastorder.com:9000
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556689 ACTION=passthru ARGS=chown otelcol:otelcol /etc/otelcol/obs-identity-sau-main-dev/config.yaml
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556698 ACTION=passthru ARGS=chmod 0640 /etc/otelcol/obs-identity-sau-main-dev/config.yaml
[0;32m[OK][0m Configuration created at /etc/otelcol/obs-identity-sau-main-dev/config.yaml
[0;34m[INFO][0m Setting up TLS certificate permissions...
[0;34m[INFO][0m Configuring certificate permissions for otlp_collector (user: otelcol)
[0;34m[INFO][0m Initializing certificate directory for obs-identity-sau-main-dev...
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556707 ACTION=passthru ARGS=chmod 755 /etc/fastorder
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556716 ACTION=passthru ARGS=chmod 755 /etc/fastorder/observability
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556725 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556734 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;32m[OK][0m Certificate directory initialized: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;34m[INFO][0m Setting file permissions...
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556744 ACTION=passthru ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-cert.pem
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556753 ACTION=passthru ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556762 ACTION=passthru ARGS=chmod 640 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-key.pem
[0;34m[INFO][0m Setting file ownership...
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556771 ACTION=passthru ARGS=chown root:otelcol /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-key.pem
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556780 ACTION=passthru ARGS=chown root:root /etc/fastorder/observability/certs/obs-identity-sau-main-dev/otlp_collector-cert.pem /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[0;34m[INFO][0m Permission configuration completed
[0;34m[INFO][0m (Verification skipped - running via wrapper, trust chmod/chown success)
[0;32m[OK][0m ✅ Certificate permissions configured successfully for otlp_collector
[0;32m[OK][0m Certificate permissions configured
[0;34m[INFO][0m Creating systemd service: otelcol-obs-identity-sau-main-dev
[0;32m[OK][0m Systemd service created at /etc/systemd/system/otelcol-obs-identity-sau-main-dev.service
[0;34m[INFO][0m Adding /etc/hosts entry for telemetry-identity-sau-main-dev-opentelemetry.fastorder.com -> 10.100.1.210
[2026-01-02 06:39:25 UTC] USER=www-data EUID=0 PID=1556800 ACTION=passthru ARGS=sed -i s/^[0-9.]*[[:space:]]*telemetry-identity-sau-main-dev-opentelemetry.fastorder.com/10.100.1.210 telemetry-identity-sau-main-dev-opentelemetry.fastorder.com/ /etc/hosts
[0;32m[OK][0m Updated /etc/hosts entry to use VM_IP
[0;34m[INFO][0m Storing OTLP configuration metadata in AWS Secrets Manager (if aws CLI present)...
{
"ARN": "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/observability/identity/sau/main/dev/otlp/collector-vA5iq3",
"Name": "fastorder/observability/identity/sau/main/dev/otlp/collector",
"VersionId": "d6e5de4f-3dd5-4236-85ba-f35ef89de37f"
}
[0;32m[OK][0m Configuration metadata stored/updated in AWS Secrets Manager: fastorder/observability/identity/sau/main/dev/otlp/collector
[0;34m[INFO][0m Enabling and starting OpenTelemetry Collector service...
[2026-01-02 06:39:28 UTC] USER=www-data EUID=0 PID=1556852 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:39:29 UTC] USER=www-data EUID=0 PID=1556908 ACTION=passthru ARGS=systemctl enable otelcol-obs-identity-sau-main-dev.service
Created symlink /etc/systemd/system/multi-user.target.wants/otelcol-obs-identity-sau-main-dev.service -> /etc/systemd/system/otelcol-obs-identity-sau-main-dev.service.
[2026-01-02 06:39:29 UTC] USER=www-data EUID=0 PID=1556958 ACTION=passthru ARGS=systemctl restart otelcol-obs-identity-sau-main-dev.service
[0;32m[OK][0m Service enabled and started
[0;34m[INFO][0m Validating deployment...
[2026-01-02 06:39:32 UTC] USER=www-data EUID=0 PID=1557002 ACTION=passthru ARGS=systemctl is-active --quiet otelcol-obs-identity-sau-main-dev.service
[0;32m[OK][0m ✅ OpenTelemetry Collector is running
[0;32m[OK][0m ✅ gRPC endpoint listening on port 4317
[0;32m[OK][0m ✅ HTTP endpoint listening on port 4318
[0;32m[OK][0m ✅ Prometheus metrics endpoint listening on port 8889
[0;34m[INFO][0m Service logs (last 10 lines):
[2026-01-02 06:39:32 UTC] USER=www-data EUID=0 PID=1557017 ACTION=passthru ARGS=journalctl -u otelcol-obs-identity-sau-main-dev.service -n 10 --no-pager
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.331Z info internal/resourcedetection.go:125 began detecting resource information {"kind": "processor", "name": "resourcedetection", "pipeline": "logs"}
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.334Z info system/system.go:201 This attribute changed from int to string. Temporarily switch back to int using the feature gate. {"kind": "processor", "name": "resourcedetection", "pipeline": "logs", "attribute": "host.cpu.family", "feature gate": "processor.resourcedetection.hostCPUModelAndFamilyAsString"}
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.336Z info system/system.go:220 This attribute changed from int to string. Temporarily switch back to int using the feature gate. {"kind": "processor", "name": "resourcedetection", "pipeline": "logs", "attribute": "host.cpu.model.id", "feature gate": "processor.resourcedetection.hostCPUModelAndFamilyAsString"}
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.336Z info internal/resourcedetection.go:139 detected resource information {"kind": "processor", "name": "resourcedetection", "pipeline": "logs", "resource": {"host.name":"web-03","os.type":"linux"}}
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.401Z info otlpreceiver@v0.91.0/otlp.go:83 Starting GRPC server {"kind": "receiver", "name": "otlp", "data_type": "traces", "endpoint": "10.100.1.210:4317"}
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.401Z info otlpreceiver@v0.91.0/otlp.go:101 Starting HTTP server {"kind": "receiver", "name": "otlp", "data_type": "traces", "endpoint": "10.100.1.210:4318"}
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.417Z info prometheusreceiver@v0.91.0/metrics_receiver.go:240 Starting discovery manager {"kind": "receiver", "name": "prometheus", "data_type": "metrics"}
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.417Z info prometheusreceiver@v0.91.0/metrics_receiver.go:231 Scrape job added {"kind": "receiver", "name": "prometheus", "data_type": "metrics", "jobName": "otel-collector"}
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.422Z info service@v0.91.0/service.go:171 Everything is ready. Begin running and processing data.
Jan 02 06:39:31 web-03 otelcol-obs-identity-sau-main-dev[1556967]: 2026-01-02T06:39:31.422Z info prometheusreceiver@v0.91.0/metrics_receiver.go:282 Starting scrape manager {"kind": "receiver", "name": "prometheus", "data_type": "metrics"}
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ Telemetry Collector Deployed Successfully
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: otlp
[0;34m[INFO][0m FQDN: telemetry-identity-sau-main-dev-opentelemetry.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.210
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Registering OpenTelemetry Collector in monitoring dashboard...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: OpenTelemetry Collector
[INFO] Identifier: identity-sau-main-dev-opentelemetry
[INFO] Identifier Parent: cluster
[INFO] IP: 10.100.1.210
[INFO] Port: 4317
[INFO] FQDN: telemetry-identity-sau-main-dev-opentelemetry.fastorder.com
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: f0ffe8a2-dfee-46fa-b427-20c818c9fa66
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[0;32m[OK][0m ✅ OpenTelemetry Collector registered in dashboard
[0;34m[INFO][0m Setting up OpenTelemetry Collector metrics collection timer...
[2026-01-02 06:39:33 UTC] USER=www-data EUID=0 PID=1557084 ACTION=passthru ARGS=mv /tmp/otelcol-metrics-identity-sau-main-dev.service /etc/systemd/system/
[2026-01-02 06:39:33 UTC] USER=www-data EUID=0 PID=1557093 ACTION=passthru ARGS=mv /tmp/otelcol-metrics-identity-sau-main-dev.timer /etc/systemd/system/
[2026-01-02 06:39:33 UTC] USER=www-data EUID=0 PID=1557102 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:39:34 UTC] USER=www-data EUID=0 PID=1557147 ACTION=passthru ARGS=systemctl enable otelcol-metrics-identity-sau-main-dev.timer
Created symlink /etc/systemd/system/timers.target.wants/otelcol-metrics-identity-sau-main-dev.timer -> /etc/systemd/system/otelcol-metrics-identity-sau-main-dev.timer.
[2026-01-02 06:39:35 UTC] USER=www-data EUID=0 PID=1557192 ACTION=passthru ARGS=systemctl start otelcol-metrics-identity-sau-main-dev.timer
[0;32m[OK][0m ✅ Metrics collection timer installed and started
[0;32m[OK][0m Telemetry collector (otlp) deployed successfully
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Step 7/10: METRICS BACKEND DEPLOYMENT
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Provider: prometheus
[0;34m[INFO][0m OBS Cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: metrics-identity-sau-main-dev-prometheus.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.205
[0;34m[INFO][0m Script: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Metrics/deploy-metrics.sh
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m 📊 METRICS DEPLOYMENT WRAPPER STARTED
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Script: deploy-metrics.sh
[0;34m[INFO][0m Timestamp: 2026-01-02 06:39:35 UTC
[0;34m[INFO][0m Arguments: --provider prometheus --obs-cell obs-identity-sau-main-dev --fqdn metrics-identity-sau-main-dev-prometheus.fastorder.com --ip 10.100.1.205
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m METRICS DEPLOYMENT
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: prometheus
[0;34m[INFO][0m Observability Cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: metrics-identity-sau-main-dev-prometheus.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.205
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Using provider: prometheus
[0;34m[INFO][0m Provider script: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Metrics/provider/prometheus.sh
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Executing provider script: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Metrics/provider/prometheus.sh
[0;34m[INFO][0m OBS_CELL: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: metrics-identity-sau-main-dev-prometheus.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.205
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m 🚀 PROMETHEUS DEPLOYMENT STARTED
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Script: prometheus.sh
[0;34m[INFO][0m Timestamp: 2026-01-02 06:39:35 UTC
[0;34m[INFO][0m Arguments: --obs-cell obs-identity-sau-main-dev --fqdn metrics-identity-sau-main-dev-prometheus.fastorder.com --ip 10.100.1.205
[0;34m[INFO][0m Parsed: SERVICE=identity, ZONE=sau, BRANCH=main, ENV=dev
[0;34m[INFO][0m Step 1/12: Sourcing centralized libraries...
[0;34m[INFO][0m Library directory: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/lib
[0;34m[INFO][0m Sourcing port_allocator.sh...
[0;32m[OK][0m ✓ port_allocator.sh loaded
[0;34m[INFO][0m Sourcing cert_permissions.sh...
[0;32m[OK][0m ✓ cert_permissions.sh loaded
[0;34m[INFO][0m Sourcing port_cleanup.sh...
[0;32m[OK][0m ✓ port_cleanup.sh loaded
[0;32m[OK][0m Step 1/12: Libraries sourced successfully
[0;34m[INFO][0m Step 2/12: Checking and cleaning ports before installation...
[0;34m[INFO][0m Initializing certificate directory for obs-identity-sau-main-dev...
[2026-01-02 06:39:35 UTC] USER=www-data EUID=0 PID=1557285 ACTION=passthru ARGS=chmod 755 /etc/fastorder
[2026-01-02 06:39:35 UTC] USER=www-data EUID=0 PID=1557309 ACTION=passthru ARGS=chmod 755 /etc/fastorder/observability
[2026-01-02 06:39:35 UTC] USER=www-data EUID=0 PID=1557318 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs
[2026-01-02 06:39:35 UTC] USER=www-data EUID=0 PID=1557329 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;32m[OK][0m Certificate directory initialized: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking and cleaning ports for observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m IP Address: 10.100.1.205
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking for conflicting observability services...
[0;34m[INFO][0m Service clickhouse-server-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service clickhouse-server-obs-identity-sau-main-dev.service belongs to current cell (skipping)
[0;34m[INFO][0m Service clickhouse-server-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-identity-sau-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-identity-sau-main-dev.service belongs to current cell (skipping)
[0;34m[INFO][0m Service otelcol-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Found 17 observability service(s) (all belong to current cell)
[0;34m[INFO][0m Checking for remaining processes on IP 10.100.1.205...
[0;34m[INFO][0m Scanning 15 ports...
[0;32m[OK][0m ✅ All 15 ports are FREE - ready for installation
[0;32m[OK][0m Port cleanup successful on attempt 1
[0;32m[OK][0m Step 2/12: Port cleanup completed
[0;34m[INFO][0m Step 3/12: Allocating ports...
[0;32m[OK][0m Step 3/12: Ports allocated
[0;34m[INFO][0m Step 4/12: Setting up configuration...
[0;34m[INFO][0m Observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: metrics-identity-sau-main-dev-prometheus.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.205
[0;34m[INFO][0m Prometheus Port: 9090
[0;34m[INFO][0m Step 5/12: Checking if Prometheus is installed...
[0;32m[OK][0m Prometheus already installed at /usr/local/bin/prometheus
[0;32m[OK][0m Step 5/12: Prometheus binary ready
[0;34m[INFO][0m Step 5.1/12: Creating configuration directories early (required for Node Exporter config)...
[0;34m[INFO][0m Config: /etc/prometheus/obs-identity-sau-main-dev
[0;34m[INFO][0m Data: /var/lib/prometheus/obs-identity-sau-main-dev
[0;34m[INFO][0m Rules: /etc/prometheus/obs-identity-sau-main-dev/rules
[2026-01-02 06:39:36 UTC] USER=www-data EUID=0 PID=1557517 ACTION=passthru ARGS=mkdir -p /etc/prometheus/obs-identity-sau-main-dev
[2026-01-02 06:39:36 UTC] USER=www-data EUID=0 PID=1557526 ACTION=passthru ARGS=mkdir -p /var/lib/prometheus/obs-identity-sau-main-dev
[2026-01-02 06:39:36 UTC] USER=www-data EUID=0 PID=1557535 ACTION=passthru ARGS=mkdir -p /etc/prometheus/obs-identity-sau-main-dev/rules
[2026-01-02 06:39:36 UTC] USER=www-data EUID=0 PID=1557544 ACTION=passthru ARGS=mkdir -p /etc/prometheus/obs-identity-sau-main-dev/targets
[0;32m[OK][0m Step 5.1/12: Directories created early
[0;34m[INFO][0m Step 6/12: Setting up Node Exporter...
[0;34m[INFO][0m Checking if Node Exporter is installed...
[0;32m[OK][0m Node Exporter already installed at /usr/local/bin/node_exporter
[0;34m[INFO][0m Creating Node Exporter TLS web config...
[0;34m[INFO][0m Creating Node Exporter systemd service with TLS...
[2026-01-02 06:39:36 UTC] USER=www-data EUID=0 PID=1557571 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:39:37 UTC] USER=www-data EUID=0 PID=1557618 ACTION=passthru ARGS=systemctl enable node_exporter-obs-identity-sau-main-dev.service
Created symlink /etc/systemd/system/multi-user.target.wants/node_exporter-obs-identity-sau-main-dev.service -> /etc/systemd/system/node_exporter-obs-identity-sau-main-dev.service.
[2026-01-02 06:39:37 UTC] USER=www-data EUID=0 PID=1557672 ACTION=passthru ARGS=systemctl restart node_exporter-obs-identity-sau-main-dev.service
[0;32m[OK][0m Step 6/12: Node Exporter ready
[0;34m[INFO][0m Step 7/12: Creating configuration directories...
[0;34m[INFO][0m Config: /etc/prometheus/obs-identity-sau-main-dev
[0;34m[INFO][0m Data: /var/lib/prometheus/obs-identity-sau-main-dev
[0;34m[INFO][0m Rules: /etc/prometheus/obs-identity-sau-main-dev/rules
[2026-01-02 06:39:37 UTC] USER=www-data EUID=0 PID=1557683 ACTION=passthru ARGS=mkdir -p /etc/prometheus/obs-identity-sau-main-dev
[2026-01-02 06:39:37 UTC] USER=www-data EUID=0 PID=1557697 ACTION=passthru ARGS=mkdir -p /var/lib/prometheus/obs-identity-sau-main-dev
[2026-01-02 06:39:37 UTC] USER=www-data EUID=0 PID=1557706 ACTION=passthru ARGS=mkdir -p /etc/prometheus/obs-identity-sau-main-dev/rules
[0;32m[OK][0m Step 7/12: Directories created
[0;34m[INFO][0m Step 8/12: Creating Prometheus configuration...
[0;34m[INFO][0m Generated FQDNs:
[0;34m[INFO][0m Prometheus: metrics-identity-sau-main-dev-prometheus.fastorder.com
[0;34m[INFO][0m Alertmanager: alerts-identity-sau-main-dev-alertmanager.fastorder.com
[0;34m[INFO][0m Grafana: dashboards-identity-sau-main-dev-grafana.fastorder.com
[0;34m[INFO][0m Otelcol: telemetry-identity-sau-main-dev-opentelemetry.fastorder.com
[0;32m[OK][0m Step 8/12: Configuration created at /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
[0;34m[INFO][0m Step 9/12: Creating TLS/HTTPS web config...
[0;32m[OK][0m Step 9/12: Web config created at /etc/prometheus/obs-identity-sau-main-dev/web-config.yml
[0;34m[INFO][0m TLS cert: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-cert.pem
[0;34m[INFO][0m TLS key: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-key.pem
[0;34m[INFO][0m CA cert: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[0;34m[INFO][0m Creating basic alerting rules...
[0;32m[OK][0m Alerting rules created
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557742 ACTION=passthru ARGS=mkdir -p /etc/prometheus/obs-identity-sau-main-dev/targets
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557751 ACTION=passthru ARGS=bash -c cat > '/etc/prometheus/obs-identity-sau-main-dev/targets/.placeholder.yml' << 'EOF'
# Placeholder file to prevent file_sd_configs warning
# Application targets will be added here automatically
[]
EOF
[0;34m[INFO][0m Step 10/12: Creating systemd service...
[0;34m[INFO][0m Service: prometheus-obs-identity-sau-main-dev
[0;34m[INFO][0m Binding to: 10.100.1.205:9090
[0;32m[OK][0m Step 10/12: Systemd service created at /etc/systemd/system/prometheus-obs-identity-sau-main-dev.service
[0;34m[INFO][0m Step 11/12: Configuring certificate permissions...
[0;34m[INFO][0m Looking for certificates in: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;32m[OK][0m ✓ All certificate files exist
[0;34m[INFO][0m Configuring certificate permissions for prometheus (user: root)
[0;34m[INFO][0m Initializing certificate directory for obs-identity-sau-main-dev...
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557770 ACTION=passthru ARGS=chmod 755 /etc/fastorder
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557779 ACTION=passthru ARGS=chmod 755 /etc/fastorder/observability
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557788 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557798 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;32m[OK][0m Certificate directory initialized: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;34m[INFO][0m Setting file permissions...
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557808 ACTION=passthru ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-cert.pem
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557826 ACTION=passthru ARGS=chmod 640 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-key.pem
[0;34m[INFO][0m Setting file ownership...
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557835 ACTION=passthru ARGS=chown root:root /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-key.pem
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557844 ACTION=passthru ARGS=chown root:root /etc/fastorder/observability/certs/obs-identity-sau-main-dev/prometheus-cert.pem /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[0;34m[INFO][0m Permission configuration completed
[0;34m[INFO][0m (Verification skipped - running via wrapper, trust chmod/chown success)
[0;32m[OK][0m ✅ Certificate permissions configured successfully for prometheus
[0;32m[OK][0m Step 11/12: Certificate permissions configured
[0;34m[INFO][0m Adding /etc/hosts entry for metrics-identity-sau-main-dev-prometheus.fastorder.com -> 10.100.1.205
[2026-01-02 06:39:38 UTC] USER=www-data EUID=0 PID=1557855 ACTION=passthru ARGS=sed -i s/^[0-9.]*[[:space:]]*metrics-identity-sau-main-dev-prometheus.fastorder.com/10.100.1.205 metrics-identity-sau-main-dev-prometheus.fastorder.com/ /etc/hosts
[0;32m[OK][0m Updated /etc/hosts entry to use VM_IP
[0;34m[INFO][0m Validating Prometheus configuration...
Checking /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
SUCCESS: 1 rule files found
SUCCESS: /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml is valid prometheus config file syntax
Checking /etc/prometheus/obs-identity-sau-main-dev/rules/basic_alerts.yml
SUCCESS: 4 rules found
[0;32m[OK][0m ✅ Configuration is valid
[0;34m[INFO][0m Storing Prometheus configuration in AWS Secrets Manager...
{
"ARN": "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/observability/identity/sau/main/dev/prometheus/server-QywcjC",
"Name": "fastorder/observability/identity/sau/main/dev/prometheus/server",
"VersionId": "062fad7d-b148-4e2b-8787-5164696bde96"
}
[0;32m[OK][0m Configuration stored in AWS Secrets Manager
[0;34m[INFO][0m Step 12/12: Starting Prometheus service...
[0;34m[INFO][0m Reloading systemd daemon...
[2026-01-02 06:39:42 UTC] USER=www-data EUID=0 PID=1557909 ACTION=passthru ARGS=systemctl daemon-reload
[0;32m[OK][0m ✓ Systemd daemon reloaded
[0;34m[INFO][0m Enabling service...
[2026-01-02 06:39:43 UTC] USER=www-data EUID=0 PID=1557961 ACTION=passthru ARGS=systemctl enable prometheus-obs-identity-sau-main-dev.service
Created symlink /etc/systemd/system/multi-user.target.wants/prometheus-obs-identity-sau-main-dev.service -> /etc/systemd/system/prometheus-obs-identity-sau-main-dev.service.
[0;32m[OK][0m ✓ Service enabled
[0;34m[INFO][0m Starting service...
[2026-01-02 06:39:43 UTC] USER=www-data EUID=0 PID=1558007 ACTION=passthru ARGS=systemctl restart prometheus-obs-identity-sau-main-dev.service
[0;32m[OK][0m ✓ Service start command issued
[0;34m[INFO][0m Validating Prometheus deployment...
[2026-01-02 06:39:46 UTC] USER=www-data EUID=0 PID=1558150 ACTION=passthru ARGS=systemctl is-active --quiet prometheus-obs-identity-sau-main-dev.service
[0;32m[OK][0m ✅ Prometheus is running
[0;32m[OK][0m ✅ Prometheus web interface listening on port 9090
[0;32m[OK][0m ✅ Prometheus health check passed (HTTPS)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m Prometheus Web UI: https://metrics-identity-sau-main-dev-prometheus.fastorder.com:9090
[0;32m[OK][0m Targets: https://metrics-identity-sau-main-dev-prometheus.fastorder.com:9090/targets
[0;32m[OK][0m Alerts: https://metrics-identity-sau-main-dev-prometheus.fastorder.com:9090/alerts
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Service logs (last 10 lines):
[2026-01-02 06:39:48 UTC] USER=www-data EUID=0 PID=1558173 ACTION=passthru ARGS=journalctl -u prometheus-obs-identity-sau-main-dev.service -n 10 --no-pager
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.382Z caller=head.go:682 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.43µs
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.382Z caller=head.go:690 level=info component=tsdb msg="Replaying WAL, this may take a while"
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.390Z caller=head.go:761 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.390Z caller=head.go:798 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=186.28µs wal_replay_duration=7.888947ms wbl_replay_duration=329ns total_replay_duration=8.118621ms
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.396Z caller=main.go:1045 level=info fs_type=EXT4_SUPER_MAGIC
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.396Z caller=main.go:1048 level=info msg="TSDB started"
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.397Z caller=main.go:1230 level=info msg="Loading configuration file" filename=/etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.404Z caller=main.go:1267 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/obs-identity-sau-main-dev/prometheus.yml totalDuration=7.922967ms db_storage=19.368µs remote_storage=4.096µs web_handler=919ns query_engine=1.87µs scrape=1.222002ms scrape_sd=521.474µs notify=240.552µs notify_sd=22.323µs rules=3.125916ms tracing=21.672µs
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.404Z caller=main.go:1009 level=info msg="Server is ready to receive web requests."
Jan 02 06:39:44 web-03 prometheus-obs-identity-sau-main-dev[1558014]: ts=2026-01-02T06:39:44.404Z caller=manager.go:1012 level=info component="rule manager" msg="Starting rule manager..."
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Provider script completed with exit code: 0
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ Metrics Deployed Successfully
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: prometheus
[0;34m[INFO][0m FQDN: metrics-identity-sau-main-dev-prometheus.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.205
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Step 7/10: METRICS DEPLOYMENT RESULT
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Exit code: 0
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m ✅ Metrics backend (prometheus) deployed successfully
[0;34m[INFO][0m Step 8/10: Deploying traces backend...
[0;34m[INFO][0m Provider: tempo (selected)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m TRACES DEPLOYMENT
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: tempo
[0;34m[INFO][0m Observability Cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: traces-identity-sau-main-dev-tempo.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.208
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Using provider: tempo
[0;34m[INFO][0m Provider script: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Traces/provider/tempo.sh
[0;34m[INFO][0m Executing provider deployment script...
[0;34m[INFO][0m Parsed: SERVICE=identity, ZONE=sau, BRANCH=main, ENV=dev
[0;34m[INFO][0m Checking and cleaning ports before installation...
[0;34m[INFO][0m Initializing certificate directory for obs-identity-sau-main-dev...
[2026-01-02 06:39:49 UTC] USER=www-data EUID=0 PID=1558193 ACTION=passthru ARGS=chmod 755 /etc/fastorder
[2026-01-02 06:39:49 UTC] USER=www-data EUID=0 PID=1558202 ACTION=passthru ARGS=chmod 755 /etc/fastorder/observability
[2026-01-02 06:39:49 UTC] USER=www-data EUID=0 PID=1558212 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs
[2026-01-02 06:39:49 UTC] USER=www-data EUID=0 PID=1558221 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;32m[OK][0m Certificate directory initialized: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking and cleaning ports for observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m IP Address: 10.100.1.208
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Checking for conflicting observability services...
[0;34m[INFO][0m Service clickhouse-server-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service clickhouse-server-obs-identity-sau-main-dev.service belongs to current cell (skipping)
[0;34m[INFO][0m Service clickhouse-server-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-identity-sau-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-metrics-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service otelcol-obs-identity-sau-main-dev.service belongs to current cell (skipping)
[0;34m[INFO][0m Service otelcol-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service prometheus-obs-identity-sau-main-dev.service belongs to current cell (skipping)
[0;34m[INFO][0m Service prometheus-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service grafana-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service tempo-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-iam-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Service alertmanager-obs-identity-universe-main-dev.service not currently listening (may be stopped or starting) - skipping
[0;34m[INFO][0m Found 18 observability service(s) (all belong to current cell)
[0;34m[INFO][0m Checking for remaining processes on IP 10.100.1.208...
[0;34m[INFO][0m Scanning 15 ports...
[0;32m[OK][0m ✅ All 15 ports are FREE - ready for installation
[0;32m[OK][0m Port cleanup successful on attempt 1
[0;34m[INFO][0m Binding Tempo to allocated IP: 10.100.1.208
[0;34m[INFO][0m Deploying Grafana Tempo for observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: traces-identity-sau-main-dev-tempo.fastorder.com
[0;34m[INFO][0m Allocated IP: 10.100.1.208
[0;34m[INFO][0m VM IP: 10.100.1.208
[0;34m[INFO][0m Ports: HTTP=3200 gRPC=9095, OTLP gRPC=4317, OTLP HTTP=4318
[0;34m[INFO][0m Checking if Grafana Tempo is installed...
[0;32m[OK][0m Grafana Tempo already installed at /usr/local/bin/tempo
[0;34m[INFO][0m Preparing configuration and data directories...
[2026-01-02 06:39:49 UTC] USER=www-data EUID=0 PID=1558409 ACTION=passthru ARGS=mkdir -p /etc/tempo/obs-identity-sau-main-dev
[2026-01-02 06:39:49 UTC] USER=www-data EUID=0 PID=1558418 ACTION=passthru ARGS=mkdir -p /var/lib/tempo/obs-identity-sau-main-dev
[2026-01-02 06:39:49 UTC] USER=www-data EUID=0 PID=1558427 ACTION=passthru ARGS=mkdir -p /var/lib/tempo/obs-identity-sau-main-dev/wal
[2026-01-02 06:39:49 UTC] USER=www-data EUID=0 PID=1558436 ACTION=passthru ARGS=mkdir -p /var/lib/tempo/obs-identity-sau-main-dev/blocks
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558445 ACTION=passthru ARGS=chown -R tempo:tempo /etc/tempo/obs-identity-sau-main-dev /var/lib/tempo/obs-identity-sau-main-dev
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558454 ACTION=passthru ARGS=chmod 750 /etc/tempo/obs-identity-sau-main-dev /var/lib/tempo/obs-identity-sau-main-dev
[0;34m[INFO][0m Creating Grafana Tempo configuration...
[0;34m[INFO][0m TLS configuration exported for tempo
[0;34m[INFO][0m Cert: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-cert.pem
[0;34m[INFO][0m Key: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-key.pem
[0;34m[INFO][0m CA: /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[0;34m[INFO][0m Setting up certificate permissions for Tempo...
[0;34m[INFO][0m Configuring certificate permissions for tempo (user: tempo)
[0;34m[INFO][0m Initializing certificate directory for obs-identity-sau-main-dev...
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558469 ACTION=passthru ARGS=chmod 755 /etc/fastorder
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558484 ACTION=passthru ARGS=chmod 755 /etc/fastorder/observability
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558493 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558502 ACTION=fsop ARGS=chmod 751 /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;32m[OK][0m Certificate directory initialized: /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;34m[INFO][0m Setting file permissions...
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558530 ACTION=passthru ARGS=chmod 640 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-key.pem
[0;34m[INFO][0m Setting file ownership...
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558539 ACTION=passthru ARGS=chown root:tempo /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-key.pem
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558548 ACTION=passthru ARGS=chown root:root /etc/fastorder/observability/certs/obs-identity-sau-main-dev/tempo-cert.pem /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[0;34m[INFO][0m Permission configuration completed
[0;34m[INFO][0m (Verification skipped - running via wrapper, trust chmod/chown success)
[0;32m[OK][0m ✅ Certificate permissions configured successfully for tempo
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558568 ACTION=passthru ARGS=chown tempo:tempo /etc/tempo/obs-identity-sau-main-dev/config.yaml
[0;32m[OK][0m Configuration created at /etc/tempo/obs-identity-sau-main-dev/config.yaml
[0;34m[INFO][0m Creating systemd service: tempo-obs-identity-sau-main-dev
[0;32m[OK][0m Systemd service created
[0;34m[INFO][0m Adding /etc/hosts entry for traces-identity-sau-main-dev-tempo.fastorder.com -> 10.100.1.208
[2026-01-02 06:39:50 UTC] USER=www-data EUID=0 PID=1558601 ACTION=passthru ARGS=sed -i s/^[0-9.]*[[:space:]]*traces-identity-sau-main-dev-tempo.fastorder.com/10.100.1.208 traces-identity-sau-main-dev-tempo.fastorder.com/ /etc/hosts
[0;32m[OK][0m Updated /etc/hosts entry to use VM_IP
[0;34m[INFO][0m Storing Tempo configuration in AWS Secrets Manager (if aws CLI present)...
{
"ARN": "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/observability/identity/sau/main/dev/tempo/server-6b9vJV",
"Name": "fastorder/observability/identity/sau/main/dev/tempo/server",
"VersionId": "8c9d5f1d-a0ae-4d08-bcdd-95f8f0d969b7"
}
[0;32m[OK][0m Tempo configuration stored/updated in AWS Secrets Manager: fastorder/observability/identity/sau/main/dev/tempo/server
[1;33m[WARN][0m Port cleanup library not found, skipping automatic cleanup
[0;34m[INFO][0m Adding iptables redirect for Tempo internal communication (optional)...
[2026-01-02 06:39:53 UTC] USER=www-data EUID=0 PID=1558632 ACTION=passthru ARGS=iptables -t nat -A OUTPUT -p tcp -d 127.0.0.1 --dport 9095 -j DNAT --to-destination 10.100.1.208:9095
ERROR: passthru not allowed: iptables
[1;33m[WARN][0m Could not add iptables redirect (iptables not allowed in wrapper)
[1;33m[WARN][0m Tempo will still work - clients should connect to 10.100.1.208:9095 directly
[0;34m[INFO][0m Enabling and starting Grafana Tempo service...
[2026-01-02 06:39:53 UTC] USER=www-data EUID=0 PID=1558645 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:39:54 UTC] USER=www-data EUID=0 PID=1558695 ACTION=passthru ARGS=systemctl enable tempo-obs-identity-sau-main-dev.service
Created symlink /etc/systemd/system/multi-user.target.wants/tempo-obs-identity-sau-main-dev.service -> /etc/systemd/system/tempo-obs-identity-sau-main-dev.service.
[2026-01-02 06:39:54 UTC] USER=www-data EUID=0 PID=1558743 ACTION=passthru ARGS=systemctl restart tempo-obs-identity-sau-main-dev.service
[0;32m[OK][0m Service enabled and started
[0;34m[INFO][0m Validating deployment...
[2026-01-02 06:39:57 UTC] USER=www-data EUID=0 PID=1558785 ACTION=passthru ARGS=systemctl is-active --quiet tempo-obs-identity-sau-main-dev.service
[0;32m[OK][0m ✅ Grafana Tempo is running
[0;32m[OK][0m ✅ HTTP endpoint listening on port 3200
[0;32m[OK][0m ✅ OTLP gRPC endpoint listening on port 4317
[0;32m[OK][0m ✅ OTLP HTTP endpoint listening on port 4318
[0;34m[INFO][0m Service logs (last 10 lines):
[2026-01-02 06:39:57 UTC] USER=www-data EUID=0 PID=1558800 ACTION=passthru ARGS=journalctl -u tempo-obs-identity-sau-main-dev.service -n 10 --no-pager
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: level=warn ts=2026-01-02T06:39:55.480364845Z caller=wal.go:94 msg="unowned file entry ignored during wal replay" file=blocks err=null
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: level=info ts=2026-01-02T06:39:55.480411345Z caller=ingester.go:402 msg="wal replay complete"
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: level=info ts=2026-01-02T06:39:55.480572904Z caller=ingester.go:416 msg="reloading local blocks" tenants=0
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: level=info ts=2026-01-02T06:39:55.480685526Z caller=lifecycler.go:624 msg="not loading tokens from file, tokens file path is empty"
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: level=info ts=2026-01-02T06:39:55.480858471Z caller=lifecycler.go:649 msg="instance not found in ring, adding with no tokens" ring=ingester
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: level=info ts=2026-01-02T06:39:55.481114437Z caller=lifecycler.go:493 msg="auto-joining cluster after timeout" ring=ingester
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: level=info ts=2026-01-02T06:39:55.48220978Z caller=worker.go:180 msg="adding connection" addr=10.100.1.208:9095
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: ts=2026-01-02T06:39:55Z level=info msg="Starting GRPC server" component=tempo endpoint=10.100.1.208:4317
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: ts=2026-01-02T06:39:55Z level=info msg="Starting HTTP server" component=tempo endpoint=10.100.1.208:4318
Jan 02 06:39:55 web-03 tempo-obs-identity-sau-main-dev[1558750]: level=info ts=2026-01-02T06:39:55.48682265Z caller=worker.go:246 msg="total worker concurrency updated" totalConcurrency=20
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ Traces Deployed Successfully
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: tempo
[0;34m[INFO][0m FQDN: traces-identity-sau-main-dev-tempo.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.208
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Registering Tempo in monitoring dashboard...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: Tempo
[INFO] Identifier: identity-sau-main-dev-tempo
[INFO] Identifier Parent: cluster
[INFO] IP: 10.100.1.208
[INFO] Port: 3200
[INFO] FQDN: traces-identity-sau-main-dev-tempo.fastorder.com
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: 16c2bd64-7e65-49ed-b269-54cb50e52ad8
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[0;32m[OK][0m ✅ Tempo registered in dashboard
[0;32m[OK][0m Traces backend (tempo) deployed successfully
[0;34m[INFO][0m Step 9/10: Deploying dashboards...
[0;34m[INFO][0m Provider: grafana (selected)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m DASHBOARDS DEPLOYMENT
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: grafana
[0;34m[INFO][0m Observability Cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: dashboards-identity-sau-main-dev-grafana.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.206
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Using provider: grafana
[0;34m[INFO][0m Provider script: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Dashboards/provider/grafana.sh
[0;34m[INFO][0m Executing provider deployment script...
[0;34m[INFO][0m Parsed: SERVICE=identity, ZONE=sau, BRANCH=main, ENV=dev
[0;34m[INFO][0m Binding to allocated IP: 10.100.1.206
[0;34m[INFO][0m Deploying Grafana for observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: dashboards-identity-sau-main-dev-grafana.fastorder.com
[0;34m[INFO][0m Allocated IP: 10.100.1.206
[0;34m[INFO][0m VM IP: 10.100.1.206
[0;34m[INFO][0m HTTP Port: 3000
[0;34m[INFO][0m Checking if Grafana is installed...
[0;32m[OK][0m Grafana already installed
[0;34m[INFO][0m Installing Grafana plugins...
[0;34m[INFO][0m Installing ClickHouse datasource plugin...
[1;33m[WARN][0m Failed to install ClickHouse plugin (may need internet access)
[0;34m[INFO][0m Validating TLS certificate and key...
[0;34m[INFO][0m Setting certificate permissions...
[0;32m[OK][0m TLS cert/key found and permissions set
[0;34m[INFO][0m Creating configuration and data directories...
[2026-01-02 06:39:58 UTC] USER=www-data EUID=0 PID=1558870 ACTION=passthru ARGS=mkdir -p /etc/grafana/obs-identity-sau-main-dev
[2026-01-02 06:39:58 UTC] USER=www-data EUID=0 PID=1558879 ACTION=passthru ARGS=mkdir -p /var/lib/grafana/obs-identity-sau-main-dev
[2026-01-02 06:39:58 UTC] USER=www-data EUID=0 PID=1558888 ACTION=passthru ARGS=mkdir -p /etc/grafana/obs-identity-sau-main-dev/provisioning/datasources
[2026-01-02 06:39:58 UTC] USER=www-data EUID=0 PID=1558897 ACTION=passthru ARGS=mkdir -p /etc/grafana/obs-identity-sau-main-dev/provisioning/dashboards
[2026-01-02 06:39:58 UTC] USER=www-data EUID=0 PID=1558906 ACTION=passthru ARGS=mkdir -p /etc/grafana/obs-identity-sau-main-dev/provisioning/notifiers
[0;34m[INFO][0m Creating Grafana configuration at /etc/grafana/obs-identity-sau-main-dev/grafana.ini...
[0;32m[OK][0m Configuration created
[0;34m[INFO][0m Creating Prometheus datasource provisioning...
[0;32m[OK][0m Prometheus datasource provisioned
[0;34m[INFO][0m Creating Tempo datasource provisioning...
[0;32m[OK][0m Tempo datasource provisioned
[0;34m[INFO][0m Creating Loki datasource provisioning...
[0;32m[OK][0m Loki datasource provisioned
[0;34m[INFO][0m Creating ClickHouse datasource provisioning...
[0;32m[OK][0m Retrieved ClickHouse credentials from Secrets Manager
[0;32m[OK][0m ClickHouse datasource provisioned
[0;34m[INFO][0m Creating systemd service: grafana-obs-identity-sau-main-dev
[0;32m[OK][0m Systemd service created
[2026-01-02 06:40:02 UTC] USER=www-data EUID=0 PID=1559072 ACTION=passthru ARGS=chown -R grafana:grafana /etc/grafana/obs-identity-sau-main-dev
[2026-01-02 06:40:02 UTC] USER=www-data EUID=0 PID=1559083 ACTION=passthru ARGS=chown -R grafana:grafana /var/lib/grafana/obs-identity-sau-main-dev
[2026-01-02 06:40:02 UTC] USER=www-data EUID=0 PID=1559096 ACTION=passthru ARGS=chmod 750 /etc/grafana/obs-identity-sau-main-dev /var/lib/grafana/obs-identity-sau-main-dev
[0;34m[INFO][0m Adding /etc/hosts entry for dashboards-identity-sau-main-dev-grafana.fastorder.com -> 10.100.1.206
[1;33m[WARN][0m /etc/hosts entry already exists
[0;34m[INFO][0m Storing Grafana credentials in AWS Secrets Manager (if aws CLI present)...
{
"ARN": "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/observability/identity/sau/main/dev/grafana/admin-nvfamv",
"Name": "fastorder/observability/identity/sau/main/dev/grafana/admin",
"VersionId": "e723e277-1c1e-485c-81c1-213045879dc1"
}
[0;32m[OK][0m Credentials stored in AWS Secrets Manager: fastorder/observability/identity/sau/main/dev/grafana/admin
[0;34m[INFO][0m Enabling and starting Grafana service...
[2026-01-02 06:40:05 UTC] USER=www-data EUID=0 PID=1559144 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:40:06 UTC] USER=www-data EUID=0 PID=1559196 ACTION=passthru ARGS=systemctl enable grafana-obs-identity-sau-main-dev.service
Created symlink /etc/systemd/system/multi-user.target.wants/grafana-obs-identity-sau-main-dev.service -> /etc/systemd/system/grafana-obs-identity-sau-main-dev.service.
[2026-01-02 06:40:07 UTC] USER=www-data EUID=0 PID=1559247 ACTION=passthru ARGS=systemctl restart grafana-obs-identity-sau-main-dev.service
[0;32m[OK][0m Service enabled and started
[0;34m[INFO][0m Validating deployment...
[2026-01-02 06:40:12 UTC] USER=www-data EUID=0 PID=1559411 ACTION=passthru ARGS=systemctl is-active --quiet grafana-obs-identity-sau-main-dev.service
[0;32m[OK][0m ✅ Grafana is running
[0;32m[OK][0m ✅ Grafana web interface listening on port 3000
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m Grafana Dashboard URL: https://dashboards-identity-sau-main-dev-grafana.fastorder.com:3000
[0;32m[OK][0m Username: admin
[0;32m[OK][0m Password is stored in AWS Secrets Manager at: fastorder/observability/identity/sau/main/dev/grafana/admin
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Service logs (last 10 lines):
[2026-01-02 06:40:12 UTC] USER=www-data EUID=0 PID=1559423 ACTION=passthru ARGS=journalctl -u grafana-obs-identity-sau-main-dev.service -n 10 --no-pager
Jan 02 06:40:11 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:11.736802622Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1"
Jan 02 06:40:11 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:11.738051511Z level=info msg="Migration successfully executed" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" duration=1.248946ms
Jan 02 06:40:11 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:11.996477225Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1"
Jan 02 06:40:12 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:12.003582873Z level=info msg="Migration successfully executed" id="Rename table dashboard to dashboard_v1 - v1" duration=7.10018ms
Jan 02 06:40:12 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:12.065009232Z level=info msg="Executing migration" id="create dashboard v2"
Jan 02 06:40:12 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:12.066248721Z level=info msg="Migration successfully executed" id="create dashboard v2" duration=1.238679ms
Jan 02 06:40:12 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:12.304271796Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2"
Jan 02 06:40:12 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:12.305411545Z level=info msg="Migration successfully executed" id="create index IDX_dashboard_org_id - v2" duration=1.141506ms
Jan 02 06:40:12 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:12.331799644Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2"
Jan 02 06:40:12 web-03 grafana-obs-identity-sau-main-dev[1559256]: logger=migrator t=2026-01-02T06:40:12.333371545Z level=info msg="Migration successfully executed" id="create index UQE_dashboard_org_id_slug - v2" duration=1.570505ms
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ Dashboards Deployed Successfully
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: grafana
[0;34m[INFO][0m FQDN: dashboards-identity-sau-main-dev-grafana.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.206
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Registering Grafana in monitoring dashboard...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: Grafana
[INFO] Identifier: identity-sau-main-dev-grafana
[INFO] Identifier Parent: cluster
[INFO] IP: 10.100.1.206
[INFO] Port: 3000
[INFO] FQDN: dashboards-identity-sau-main-dev-grafana.fastorder.com
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: aa1bcd01-48c6-4a69-a9e1-c9cb1ce8202a
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[0;32m[OK][0m ✅ Grafana registered in dashboard
[0;32m[OK][0m Dashboards (grafana) deployed successfully
[0;34m[INFO][0m Step 10/10: Deploying alerting...
[0;34m[INFO][0m Provider: alertmanager (selected)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m ALERTING DEPLOYMENT
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: alertmanager
[0;34m[INFO][0m Observability Cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: alerts-identity-sau-main-dev-alertmanager.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.209
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Ports: Web=9093 Cluster=9094 (bound to IP: 10.100.1.209)
[0;34m[INFO][0m Using provider: alertmanager
[0;34m[INFO][0m Provider script: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Alerting/provider/alertmanager.sh
[0;34m[INFO][0m Executing provider deployment script...
[0;34m[INFO][0m Parsed: SERVICE=identity, ZONE=sau, BRANCH=main, ENV=dev
[0;34m[INFO][0m Binding to allocated IP: 10.100.1.209
[0;34m[INFO][0m Deploying Alertmanager for observability cell: obs-identity-sau-main-dev
[0;34m[INFO][0m FQDN: alerts-identity-sau-main-dev-alertmanager.fastorder.com
[0;34m[INFO][0m Allocated IP: 10.100.1.209
[0;34m[INFO][0m VM IP: 10.100.1.209
[0;34m[INFO][0m Ports: Web=9093 Cluster=9094
[0;34m[INFO][0m Checking if Alertmanager is installed...
[0;32m[OK][0m Alertmanager already installed at /usr/local/bin/alertmanager
[0;34m[INFO][0m Validating TLS certificate and key...
[0;32m[OK][0m TLS cert/key found in /etc/fastorder/observability/certs/obs-identity-sau-main-dev
[0;34m[INFO][0m Creating configuration and data directories...
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559509 ACTION=passthru ARGS=mkdir -p /var/lib/alertmanager/obs-identity-sau-main-dev
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559518 ACTION=passthru ARGS=mkdir -p /etc/alertmanager/obs-identity-sau-main-dev/templates
[0;34m[INFO][0m Creating Alertmanager configuration...
[0;32m[OK][0m Alertmanager configuration created at /etc/alertmanager/obs-identity-sau-main-dev/alertmanager.yml
[0;34m[INFO][0m Creating notification templates...
[0;32m[OK][0m Notification templates created
[0;34m[INFO][0m Creating Alertmanager web TLS configuration with mTLS...
[0;32m[OK][0m Web mTLS configuration created at /etc/alertmanager/obs-identity-sau-main-dev/web-config.yml
[0;34m[INFO][0m Validating Alertmanager configuration...
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559557 ACTION=passthru ARGS=chmod 755 /etc/alertmanager/obs-identity-sau-main-dev
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559566 ACTION=passthru ARGS=chmod 644 /etc/alertmanager/obs-identity-sau-main-dev/alertmanager.yml
Checking '/etc/alertmanager/obs-identity-sau-main-dev/alertmanager.yml' SUCCESS
Found:
- global config
- route
- 6 inhibit rules
- 5 receivers
- 1 templates
SUCCESS
[0;32m[OK][0m ✅ Configuration is valid
[0;34m[INFO][0m Creating systemd service: alertmanager-obs-identity-sau-main-dev
[0;32m[OK][0m Systemd service created
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559593 ACTION=passthru ARGS=chown alertmanager:alertmanager /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-key.pem
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559602 ACTION=passthru ARGS=chown alertmanager:alertmanager /etc/fastorder/observability/certs/obs-identity-sau-main-dev/alertmanager-cert.pem
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559611 ACTION=passthru ARGS=chmod 644 /etc/fastorder/observability/certs/obs-identity-sau-main-dev/ca-cert.pem
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559620 ACTION=passthru ARGS=chown -R alertmanager:alertmanager /etc/alertmanager/obs-identity-sau-main-dev
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559629 ACTION=passthru ARGS=chown -R alertmanager:alertmanager /var/lib/alertmanager/obs-identity-sau-main-dev
[2026-01-02 06:40:14 UTC] USER=www-data EUID=0 PID=1559638 ACTION=passthru ARGS=chmod 750 /etc/alertmanager/obs-identity-sau-main-dev /var/lib/alertmanager/obs-identity-sau-main-dev
[0;34m[INFO][0m Adding /etc/hosts entry for alerts-identity-sau-main-dev-alertmanager.fastorder.com -> 10.100.1.209
[1;33m[WARN][0m /etc/hosts entry already exists
[0;34m[INFO][0m Storing Alertmanager configuration in AWS Secrets Manager (if aws CLI present)...
{
"ARN": "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/observability/identity/sau/main/dev/alertmanager/server-igfNGa",
"Name": "fastorder/observability/identity/sau/main/dev/alertmanager/server",
"VersionId": "60b59ced-15fd-4706-891b-b5a622fef2cd"
}
[0;32m[OK][0m Configuration stored in AWS Secrets Manager: fastorder/observability/identity/sau/main/dev/alertmanager/server
[0;34m[INFO][0m Enabling and starting Alertmanager service...
[2026-01-02 06:40:18 UTC] USER=www-data EUID=0 PID=1559674 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:40:18 UTC] USER=www-data EUID=0 PID=1559726 ACTION=passthru ARGS=systemctl enable alertmanager-obs-identity-sau-main-dev.service
Created symlink /etc/systemd/system/multi-user.target.wants/alertmanager-obs-identity-sau-main-dev.service -> /etc/systemd/system/alertmanager-obs-identity-sau-main-dev.service.
[2026-01-02 06:40:19 UTC] USER=www-data EUID=0 PID=1559772 ACTION=passthru ARGS=systemctl restart alertmanager-obs-identity-sau-main-dev.service
[0;32m[OK][0m Service enabled and started
[0;34m[INFO][0m Validating deployment...
[2026-01-02 06:40:22 UTC] USER=www-data EUID=0 PID=1559816 ACTION=passthru ARGS=systemctl is-active --quiet alertmanager-obs-identity-sau-main-dev.service
[0;32m[OK][0m ✅ Alertmanager is running
[0;32m[OK][0m ✅ Alertmanager HTTPS web interface listening on port 9093
[0;32m[OK][0m ✅ Alertmanager cluster port listening on port 9094
[1;33m[WARN][0m ⚠️ Alertmanager health check not responding yet (HTTPS)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m Alertmanager Web UI: https://alerts-identity-sau-main-dev-alertmanager.fastorder.com:9093
[0;32m[OK][0m API Endpoint: https://alerts-identity-sau-main-dev-alertmanager.fastorder.com:9093/api/v2
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Service logs (last 10 lines):
[2026-01-02 06:40:22 UTC] USER=www-data EUID=0 PID=1559830 ACTION=passthru ARGS=journalctl -u alertmanager-obs-identity-sau-main-dev.service -n 10 --no-pager
Jan 02 06:40:19 web-03 systemd[1]: Started Alertmanager - obs-identity-sau-main-dev.
Jan 02 06:40:19 web-03 alertmanager-obs-identity-sau-main-dev[1559779]: ts=2026-01-02T06:40:19.949Z caller=main.go:245 level=info msg="Starting Alertmanager" version="(version=0.26.0, branch=HEAD, revision=d7b4f0c7322e7151d6e3b1e31cbc15361e295d8d)"
Jan 02 06:40:19 web-03 alertmanager-obs-identity-sau-main-dev[1559779]: ts=2026-01-02T06:40:19.954Z caller=main.go:246 level=info build_context="(go=go1.20.7, platform=linux/amd64, user=root@df8d7debeef4, date=20230824-11:11:58, tags=netgo)"
Jan 02 06:40:19 web-03 alertmanager-obs-identity-sau-main-dev[1559779]: ts=2026-01-02T06:40:19.969Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s
Jan 02 06:40:20 web-03 alertmanager-obs-identity-sau-main-dev[1559779]: ts=2026-01-02T06:40:20.078Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/obs-identity-sau-main-dev/alertmanager.yml
Jan 02 06:40:20 web-03 alertmanager-obs-identity-sau-main-dev[1559779]: ts=2026-01-02T06:40:20.084Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/obs-identity-sau-main-dev/alertmanager.yml
Jan 02 06:40:20 web-03 alertmanager-obs-identity-sau-main-dev[1559779]: ts=2026-01-02T06:40:20.097Z caller=tls_config.go:274 level=info msg="Listening on" address=10.100.1.209:9093
Jan 02 06:40:20 web-03 alertmanager-obs-identity-sau-main-dev[1559779]: ts=2026-01-02T06:40:20.099Z caller=tls_config.go:310 level=info msg="TLS is enabled." http2=true address=10.100.1.209:9093
Jan 02 06:40:21 web-03 alertmanager-obs-identity-sau-main-dev[1559779]: ts=2026-01-02T06:40:21.970Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00045416s
Jan 02 06:40:22 web-03 alertmanager-obs-identity-sau-main-dev[1559779]: 2026/01/02 06:40:22 http: TLS handshake error from 10.100.1.209:53664: tls: client didn't provide a certificate
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ Alerting Deployed Successfully
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Provider: alertmanager
[0;34m[INFO][0m FQDN: alerts-identity-sau-main-dev-alertmanager.fastorder.com
[0;34m[INFO][0m IP: 10.100.1.209
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Setting up HTTPS reverse proxy...
[0;34m[INFO][0m Backend port: 9093
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Alertmanager HTTPS Reverse Proxy Setup
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
OBS Cell: obs-identity-sau-main-dev
FQDN: alerts-identity-sau-main-dev-alertmanager.fastorder.com
Backend: https://alerts-identity-sau-main-dev-alertmanager.fastorder.com:9093/ (resolved via /etc/hosts)
Backend IP: 10.100.1.209
Email: admin@fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Verifying prerequisites...
[0;31m[ERROR][0m This script must be run as root or with sudo
[1;33m[WARN][0m ⚠️ HTTPS setup failed (Alertmanager is still running on HTTP)
[0;34m[INFO][0m Registering Alertmanager in monitoring dashboard...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: Alertmanager
[INFO] Identifier: identity-sau-main-dev-alertmanager
[INFO] Identifier Parent: cluster
[INFO] IP: 10.100.1.209
[INFO] Port: 9093
[INFO] FQDN: alerts-identity-sau-main-dev-alertmanager.fastorder.com
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: 94edbca8-b241-48c0-add2-e95dc8cd9fe2
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[0;32m[OK][0m ✅ Alertmanager registered in dashboard
[0;32m[OK][0m Alerting (alertmanager) deployed successfully
[0;34m[INFO][0m Step 10.5: Deploying Blackbox Exporter for synthetic monitoring...
[0;32m[BLACKBOX][0m Starting Blackbox Exporter deployment for obs-identity-sau-main-dev
[0;32m[BLACKBOX][0m VM IP: 10.100.1.205
[0;32m[BLACKBOX][0m Version: 0.25.0
[0;32m[BLACKBOX][0m Checking prerequisites...
[0;32m[BLACKBOX][0m Creating directories...
[0;32m[BLACKBOX][0m Downloading Blackbox Exporter v0.25.0...
Sorry, user www-data is not allowed to execute '/usr/bin/mv /tmp/tmp.ToL4J3Mcnt/blackbox_exporter-0.25.0.linux-amd64/blackbox_exporter /usr/local/bin/' as root on web-03.
[1;33m[WARN][0m Blackbox Exporter deployment failed (non-fatal, synthetic monitoring disabled)
[0;34m[INFO][0m Step 11/13: Configuring HTTPS reverse proxies...
[0;34m[INFO][0m Setting up Prometheus HTTPS proxy...
[2026-01-02 06:40:24 UTC] USER=www-data EUID=0 PID=1559924 ACTION=passthru ARGS=bash /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Metrics/https/setup-prometheus-https.sh --obs-cell obs-identity-sau-main-dev --backend-ip 10.100.1.205
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Prometheus HTTPS Reverse Proxy Setup
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
OBS Cell: obs-identity-sau-main-dev
FQDN: metrics-identity-sau-main-dev-prometheus.fastorder.com
Backend: https://metrics-identity-sau-main-dev-prometheus.fastorder.com:9090/ (resolved via /etc/hosts)
Backend IP: 10.100.1.205
Email: admin@fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Verifying prerequisites...
[0;34m[INFO][0m Enabling Apache modules...
[0;34m[INFO][0m Testing backend connectivity (will retry up to 60s)...
[0;32m[OK][0m Backend is accessible
[0;34m[INFO][0m Creating HTTP VirtualHost for ACME challenge...
[0;32m[OK][0m HTTP VirtualHost created
[0;34m[INFO][0m Obtaining Let's Encrypt certificate...
Saving debug log to /var/log/letsencrypt/letsencrypt.log
Requesting a certificate for metrics-identity-sau-main-dev-prometheus.fastorder.com
Successfully received certificate.
Certificate is saved at: /etc/letsencrypt/live/metrics-identity-sau-main-dev-prometheus.fastorder.com/fullchain.pem
Key is saved at: /etc/letsencrypt/live/metrics-identity-sau-main-dev-prometheus.fastorder.com/privkey.pem
This certificate expires on 2026-04-02.
These files will be updated when the certificate renews.
NEXT STEPS:
- The certificate will need to be renewed before it expires. Certbot can automatically renew the certificate in the background, but you may need to take steps to enable that functionality. See https://certbot.org/renewal-setup for instructions.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
If you like Certbot, please consider supporting our work by:
* Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate
* Donating to EFF: https://eff.org/donate-le
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
[0;32m[OK][0m Certificate obtained
[0;34m[INFO][0m Creating HTTPS VirtualHost with reverse proxy...
[0;32m[OK][0m HTTPS VirtualHost created and Apache reloaded
[0;34m[INFO][0m Setting up certificate auto-renewal...
[0;32m[OK][0m Auto-renewal configured
[0;34m[INFO][0m Updating /etc/hosts...
[0;32m[OK][0m /etc/hosts updated
[0;34m[INFO][0m Verifying HTTPS setup...
[0;32m[OK][0m HTTPS endpoint is working
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m✅ Prometheus HTTPS Setup Complete[0m
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
HTTPS Endpoints:
Health: https://metrics-identity-sau-main-dev-prometheus.fastorder.com/-/healthy
Ready: https://metrics-identity-sau-main-dev-prometheus.fastorder.com/-/ready
Graph: https://metrics-identity-sau-main-dev-prometheus.fastorder.com/graph
Targets: https://metrics-identity-sau-main-dev-prometheus.fastorder.com/targets
Alerts: https://metrics-identity-sau-main-dev-prometheus.fastorder.com/alerts
API: https://metrics-identity-sau-main-dev-prometheus.fastorder.com/api/v1/...
Apache VirtualHosts:
HTTP: /etc/apache2/sites-available/metrics-identity-sau-main-dev-prometheus.fastorder.com.conf
HTTPS: /etc/apache2/sites-available/metrics-identity-sau-main-dev-prometheus.fastorder.com-ssl.conf
Certificate:
Path: /etc/letsencrypt/live/metrics-identity-sau-main-dev-prometheus.fastorder.com/
Renewal: certbot renew --cert-name metrics-identity-sau-main-dev-prometheus.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m Prometheus HTTPS proxy configured
[0;34m[INFO][0m Setting up Grafana HTTPS proxy...
[2026-01-02 06:40:42 UTC] USER=www-data EUID=0 PID=1560324 ACTION=passthru ARGS=bash /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Dashboards/https/setup-grafana-https.sh --obs-cell obs-identity-sau-main-dev --backend-ip 10.100.1.206
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Grafana HTTPS Reverse Proxy Setup
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
OBS Cell: obs-identity-sau-main-dev
FQDN: dashboards-identity-sau-main-dev-grafana.fastorder.com
Backend: https://dashboards-identity-sau-main-dev-grafana.fastorder.com:3000/ (resolved via /etc/hosts)
Backend IP: 10.100.1.206
Email: admin@fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Verifying prerequisites...
[0;34m[INFO][0m Enabling Apache modules...
[0;34m[INFO][0m Testing backend connectivity...
[0;34m[INFO][0m Creating HTTP VirtualHost for ACME challenge...
[0;32m[OK][0m HTTP VirtualHost created
[0;34m[INFO][0m Obtaining Let's Encrypt certificate...
Saving debug log to /var/log/letsencrypt/letsencrypt.log
Requesting a certificate for dashboards-identity-sau-main-dev-grafana.fastorder.com
Successfully received certificate.
Certificate is saved at: /etc/letsencrypt/live/dashboards-identity-sau-main-dev-grafana.fastorder.com/fullchain.pem
Key is saved at: /etc/letsencrypt/live/dashboards-identity-sau-main-dev-grafana.fastorder.com/privkey.pem
This certificate expires on 2026-04-02.
These files will be updated when the certificate renews.
NEXT STEPS:
- The certificate will need to be renewed before it expires. Certbot can automatically renew the certificate in the background, but you may need to take steps to enable that functionality. See https://certbot.org/renewal-setup for instructions.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
If you like Certbot, please consider supporting our work by:
* Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate
* Donating to EFF: https://eff.org/donate-le
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
[0;32m[OK][0m Certificate obtained
[0;34m[INFO][0m Creating HTTPS VirtualHost...
[0;32m[OK][0m HTTPS VirtualHost created and Apache reloaded
[0;34m[INFO][0m Updating /etc/hosts...
[0;32m[OK][0m /etc/hosts updated
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Grafana HTTPS Setup Complete!
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Grafana URL: https://dashboards-identity-sau-main-dev-grafana.fastorder.com/
Metrics: https://dashboards-identity-sau-main-dev-grafana.fastorder.com/metrics
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m Grafana HTTPS proxy configured
[0;34m[INFO][0m Setting up OpenTelemetry Collector HTTPS proxy...
[2026-01-02 06:40:53 UTC] USER=www-data EUID=0 PID=1560595 ACTION=passthru ARGS=bash /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Telemetry/https/setup-otelcol-https.sh --obs-cell obs-identity-sau-main-dev --backend-ip 10.100.1.210
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
OpenTelemetry Collector HTTPS Reverse Proxy Setup
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
OBS Cell: obs-identity-sau-main-dev
FQDN: telemetry-identity-sau-main-dev-opentelemetry.fastorder.com
Backend: http://telemetry-identity-sau-main-dev-opentelemetry.fastorder.com:8888/ (resolved via /etc/hosts)
Backend IP: 10.100.1.210
Email: admin@fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Verifying prerequisites...
[0;34m[INFO][0m Enabling Apache modules...
[0;34m[INFO][0m Testing backend connectivity...
[0;32m[OK][0m Backend is accessible and returning metrics via HTTPS
[0;34m[INFO][0m Creating HTTP VirtualHost for ACME challenge...
[0;32m[OK][0m HTTP VirtualHost created
[0;34m[INFO][0m Obtaining Let's Encrypt certificate...
Saving debug log to /var/log/letsencrypt/letsencrypt.log
Requesting a certificate for telemetry-identity-sau-main-dev-opentelemetry.fastorder.com
Successfully received certificate.
Certificate is saved at: /etc/letsencrypt/live/telemetry-identity-sau-main-dev-opentelemetry.fastorder.com/fullchain.pem
Key is saved at: /etc/letsencrypt/live/telemetry-identity-sau-main-dev-opentelemetry.fastorder.com/privkey.pem
This certificate expires on 2026-04-02.
These files will be updated when the certificate renews.
NEXT STEPS:
- The certificate will need to be renewed before it expires. Certbot can automatically renew the certificate in the background, but you may need to take steps to enable that functionality. See https://certbot.org/renewal-setup for instructions.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
If you like Certbot, please consider supporting our work by:
* Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate
* Donating to EFF: https://eff.org/donate-le
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
[0;32m[OK][0m Certificate obtained
[0;34m[INFO][0m Creating HTTPS VirtualHost with reverse proxy...
[0;32m[OK][0m HTTPS VirtualHost created and Apache reloaded
[0;34m[INFO][0m Setting up certificate auto-renewal...
[0;32m[OK][0m Auto-renewal configured
[0;34m[INFO][0m Updating /etc/hosts...
[0;32m[OK][0m /etc/hosts updated
[0;34m[INFO][0m Verifying HTTPS setup...
[1;33m[WARN][0m HTTPS verification failed - may need DNS propagation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32mOpenTelemetry Collector HTTPS Setup Complete[0m
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
HTTPS Endpoints:
Metrics: https://telemetry-identity-sau-main-dev-opentelemetry.fastorder.com/metrics
Apache VirtualHosts:
HTTP: /etc/apache2/sites-available/telemetry-identity-sau-main-dev-opentelemetry.fastorder.com.conf
HTTPS: /etc/apache2/sites-available/telemetry-identity-sau-main-dev-opentelemetry.fastorder.com-ssl.conf
Certificate:
Path: /etc/letsencrypt/live/telemetry-identity-sau-main-dev-opentelemetry.fastorder.com/
Renewal: certbot renew --cert-name telemetry-identity-sau-main-dev-opentelemetry.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m OpenTelemetry Collector HTTPS proxy configured
[0;34m[INFO][0m Setting up ClickHouse HTTPS proxy...
[2026-01-02 06:41:08 UTC] USER=www-data EUID=0 PID=1560886 ACTION=passthru ARGS=bash /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/LogStorageBackend/https/setup-clickhouse-https.sh --obs-cell obs-identity-sau-main-dev --backend-ip 10.100.1.207
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
ClickHouse HTTPS Reverse Proxy Setup
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
OBS Cell: obs-identity-sau-main-dev
FQDN: logstore-identity-sau-main-dev.fastorder.com
Backend: http://logstore-identity-sau-main-dev.fastorder.com:8123/ (resolved via /etc/hosts)
Backend IP: 10.100.1.207
Email: admin@fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Verifying prerequisites...
[0;34m[INFO][0m Enabling Apache modules...
[0;34m[INFO][0m Testing backend connectivity (will retry up to 60s)...
[0;32m[OK][0m Backend is accessible
[0;34m[INFO][0m Creating HTTP VirtualHost for ACME challenge...
[0;32m[OK][0m HTTP VirtualHost created
[0;34m[INFO][0m Obtaining Let's Encrypt certificate...
Saving debug log to /var/log/letsencrypt/letsencrypt.log
Requesting a certificate for logstore-identity-sau-main-dev.fastorder.com
Successfully received certificate.
Certificate is saved at: /etc/letsencrypt/live/logstore-identity-sau-main-dev.fastorder.com/fullchain.pem
Key is saved at: /etc/letsencrypt/live/logstore-identity-sau-main-dev.fastorder.com/privkey.pem
This certificate expires on 2026-04-02.
These files will be updated when the certificate renews.
NEXT STEPS:
- The certificate will need to be renewed before it expires. Certbot can automatically renew the certificate in the background, but you may need to take steps to enable that functionality. See https://certbot.org/renewal-setup for instructions.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
If you like Certbot, please consider supporting our work by:
* Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate
* Donating to EFF: https://eff.org/donate-le
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
[0;32m[OK][0m Certificate obtained
[0;34m[INFO][0m Creating HTTPS VirtualHost with reverse proxy...
[0;32m[OK][0m HTTPS VirtualHost created and Apache reloaded
[0;34m[INFO][0m Setting up certificate auto-renewal...
[0;32m[OK][0m Auto-renewal configured
[0;34m[INFO][0m Updating /etc/hosts...
[0;32m[OK][0m /etc/hosts updated
[0;34m[INFO][0m Verifying HTTPS setup...
[0;32m[OK][0m HTTPS endpoint is working
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m✅ ClickHouse HTTPS Setup Complete[0m
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
HTTPS Endpoints:
Health: https://logstore-identity-sau-main-dev.fastorder.com/
Dashboard: https://logstore-identity-sau-main-dev.fastorder.com/dashboard
Playground: https://logstore-identity-sau-main-dev.fastorder.com/play
Metrics: https://logstore-identity-sau-main-dev.fastorder.com/metrics
Login Instructions:
1. Get credentials from skeleton: POST /api/monitoring/clickhouse/credentials
2. Use auto-login URL: https://logstore-identity-sau-main-dev.fastorder.com/dashboard#user=<USER>&password=<PASS>
3. Or use skeleton monitoring dashboard for one-click access
Apache VirtualHosts:
HTTP: /etc/apache2/sites-available/logstore-identity-sau-main-dev.fastorder.com.conf
HTTPS: /etc/apache2/sites-available/logstore-identity-sau-main-dev.fastorder.com-ssl.conf
Certificate:
Path: /etc/letsencrypt/live/logstore-identity-sau-main-dev.fastorder.com/
Auto-renewal: Enabled via certbot.timer
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m ClickHouse HTTPS proxy configured
[0;34m[INFO][0m Setting up Tempo HTTPS proxy...
[2026-01-02 06:41:24 UTC] USER=www-data EUID=0 PID=1561273 ACTION=passthru ARGS=bash /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Traces/https/setup-tempo-https.sh --obs-cell obs-identity-sau-main-dev --backend-ip 10.100.1.208
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Grafana Tempo HTTPS Reverse Proxy Setup
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
OBS Cell: obs-identity-sau-main-dev
FQDN: traces-identity-sau-main-dev-tempo.fastorder.com
Backend: https://10.100.1.208:3200/
Backend IP: 10.100.1.208
Email: admin@fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Verifying prerequisites...
[0;34m[INFO][0m Enabling Apache modules...
[0;34m[INFO][0m Testing backend connectivity...
[1;33m[WARN][0m Cannot verify Tempo health endpoint (it may not be running yet), continuing anyway...
[0;34m[INFO][0m Creating HTTP VirtualHost for ACME challenge...
[0;32m[OK][0m HTTP VirtualHost created
[0;34m[INFO][0m Obtaining Let's Encrypt certificate...
Saving debug log to /var/log/letsencrypt/letsencrypt.log
Requesting a certificate for traces-identity-sau-main-dev-tempo.fastorder.com
Successfully received certificate.
Certificate is saved at: /etc/letsencrypt/live/traces-identity-sau-main-dev-tempo.fastorder.com/fullchain.pem
Key is saved at: /etc/letsencrypt/live/traces-identity-sau-main-dev-tempo.fastorder.com/privkey.pem
This certificate expires on 2026-04-02.
These files will be updated when the certificate renews.
NEXT STEPS:
- The certificate will need to be renewed before it expires. Certbot can automatically renew the certificate in the background, but you may need to take steps to enable that functionality. See https://certbot.org/renewal-setup for instructions.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
If you like Certbot, please consider supporting our work by:
* Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate
* Donating to EFF: https://eff.org/donate-le
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
[0;32m[OK][0m Certificate obtained
[0;34m[INFO][0m Generating Apache client certificate for mTLS backend connection...
[0;32m[OK][0m Apache client certificate already exists
[0;34m[INFO][0m Creating HTTPS VirtualHost with mTLS backend...
[0;32m[OK][0m HTTPS VirtualHost created and Apache reloaded
[0;34m[INFO][0m Updating /etc/hosts...
[0;32m[OK][0m /etc/hosts updated
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Grafana Tempo HTTPS Setup Complete!
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Tempo URL: https://traces-identity-sau-main-dev-tempo.fastorder.com/
Ready: https://traces-identity-sau-main-dev-tempo.fastorder.com/ready
Metrics: https://traces-identity-sau-main-dev-tempo.fastorder.com/metrics
Build Info: https://traces-identity-sau-main-dev-tempo.fastorder.com/api/status/buildinfo
Note: Tempo backend must be running at traces-identity-sau-main-dev-tempo.fastorder.com:3200 (10.100.1.208)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m Tempo HTTPS proxy configured
[0;34m[INFO][0m Setting up Alertmanager HTTPS proxy...
[2026-01-02 06:41:39 UTC] USER=www-data EUID=0 PID=1561625 ACTION=passthru ARGS=bash /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/Alerting/https/setup-alertmanager-https.sh --obs-cell obs-identity-sau-main-dev --backend-ip 10.100.1.209
[0;34m[INFO][0m Backend port: 9093
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Alertmanager HTTPS Reverse Proxy Setup
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
OBS Cell: obs-identity-sau-main-dev
FQDN: alerts-identity-sau-main-dev-alertmanager.fastorder.com
Backend: https://alerts-identity-sau-main-dev-alertmanager.fastorder.com:9093/ (resolved via /etc/hosts)
Backend IP: 10.100.1.209
Email: admin@fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Verifying prerequisites...
[0;34m[INFO][0m Enabling Apache modules...
[0;34m[INFO][0m Testing backend connectivity...
[1;33m[WARN][0m Backend health check inconclusive - proceeding anyway
[0;34m[INFO][0m Creating HTTP VirtualHost for ACME challenge...
[0;32m[OK][0m HTTP VirtualHost created
[0;34m[INFO][0m Obtaining Let's Encrypt certificate...
Saving debug log to /var/log/letsencrypt/letsencrypt.log
Requesting a certificate for alerts-identity-sau-main-dev-alertmanager.fastorder.com
Successfully received certificate.
Certificate is saved at: /etc/letsencrypt/live/alerts-identity-sau-main-dev-alertmanager.fastorder.com/fullchain.pem
Key is saved at: /etc/letsencrypt/live/alerts-identity-sau-main-dev-alertmanager.fastorder.com/privkey.pem
This certificate expires on 2026-04-02.
These files will be updated when the certificate renews.
NEXT STEPS:
- The certificate will need to be renewed before it expires. Certbot can automatically renew the certificate in the background, but you may need to take steps to enable that functionality. See https://certbot.org/renewal-setup for instructions.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
If you like Certbot, please consider supporting our work by:
* Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate
* Donating to EFF: https://eff.org/donate-le
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
[0;32m[OK][0m Certificate obtained
[0;34m[INFO][0m Creating HTTPS VirtualHost with reverse proxy...
[0;32m[OK][0m HTTPS VirtualHost created and Apache reloaded
[0;34m[INFO][0m Setting up certificate auto-renewal...
[0;32m[OK][0m Auto-renewal configured
[0;34m[INFO][0m Updating /etc/hosts...
[0;32m[OK][0m /etc/hosts updated
[0;34m[INFO][0m Verifying HTTPS setup...
OK[0;32m[OK][0m HTTPS endpoint is working
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m✅ Alertmanager HTTPS Setup Complete[0m
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
HTTPS Endpoints:
Health: https://alerts-identity-sau-main-dev-alertmanager.fastorder.com/-/healthy
Ready: https://alerts-identity-sau-main-dev-alertmanager.fastorder.com/-/ready
Web UI: https://alerts-identity-sau-main-dev-alertmanager.fastorder.com/
API: https://alerts-identity-sau-main-dev-alertmanager.fastorder.com/api/v2/...
Apache VirtualHosts:
HTTP: /etc/apache2/sites-available/alerts-identity-sau-main-dev-alertmanager.fastorder.com.conf
HTTPS: /etc/apache2/sites-available/alerts-identity-sau-main-dev-alertmanager.fastorder.com-ssl.conf
Certificate:
Path: /etc/letsencrypt/live/alerts-identity-sau-main-dev-alertmanager.fastorder.com/
Renewal: certbot renew --cert-name alerts-identity-sau-main-dev-alertmanager.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m Alertmanager HTTPS proxy configured
[0;32m[OK][0m HTTPS reverse proxies configured
[0;34m[INFO][0m Step 12/13: Configuring firewall rules (network segmentation)...
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m CONFIGURING FIREWALL RULES FOR OBSERVABILITY CELL
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Cell ID: obs-identity-sau-main-dev
[0;34m[INFO][0m Internal Network: 10.0.0.0/8
[0;34m[INFO][0m Discovering dashboard/skeleton VM IPs...
[0;34m[INFO][0m Discovered skeleton IP: 142.93.238.16 (skeleton.fastorder.com)
[0;34m[INFO][0m Authorized dashboard IPs:
[0;34m[INFO][0m - 10.100.60.2
[0;34m[INFO][0m - 142.93.238.16
[0;34m[INFO][0m Configuring UFW firewall rules...
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562061 ACTION=passthru ARGS=ufw --force enable
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562069 ACTION=passthru ARGS=ufw default deny incoming
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562078 ACTION=passthru ARGS=ufw default allow outgoing
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562086 ACTION=passthru ARGS=ufw allow 22/tcp comment SSH
ERROR: passthru not allowed: ufw
[0;34m[INFO][0m Allowing prometheus (port 9090) from internal network...
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562094 ACTION=passthru ARGS=ufw allow from 10.0.0.0/8 to any port 9090 proto tcp comment Obs: prometheus from internal
ERROR: passthru not allowed: ufw
[0;34m[INFO][0m Allowing alertmanager (port 9093) from internal network...
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562102 ACTION=passthru ARGS=ufw allow from 10.0.0.0/8 to any port 9093 proto tcp comment Obs: alertmanager from internal
ERROR: passthru not allowed: ufw
[0;34m[INFO][0m Allowing clickhouse (port 8123) from internal network...
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562110 ACTION=passthru ARGS=ufw allow from 10.0.0.0/8 to any port 8123 proto tcp comment Obs: clickhouse from internal
ERROR: passthru not allowed: ufw
[0;34m[INFO][0m Allowing grafana (port 3000) from internal network...
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562118 ACTION=passthru ARGS=ufw allow from 10.0.0.0/8 to any port 3000 proto tcp comment Obs: grafana from internal
ERROR: passthru not allowed: ufw
[0;34m[INFO][0m Allowing otelcol (port 4318) from internal network...
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562128 ACTION=passthru ARGS=ufw allow from 10.0.0.0/8 to any port 4318 proto tcp comment Obs: otelcol from internal
ERROR: passthru not allowed: ufw
[0;34m[INFO][0m Allowing loki (port 3100) from internal network...
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562136 ACTION=passthru ARGS=ufw allow from 10.0.0.0/8 to any port 3100 proto tcp comment Obs: loki from internal
ERROR: passthru not allowed: ufw
[0;34m[INFO][0m Allowing tempo (port 3200) from internal network...
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562144 ACTION=passthru ARGS=ufw allow from 10.0.0.0/8 to any port 3200 proto tcp comment Obs: tempo from internal
ERROR: passthru not allowed: ufw
[0;34m[INFO][0m Allowing dashboard access from 10.100.60.2...
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562152 ACTION=passthru ARGS=ufw allow from 10.100.60.2 to any port 9090 proto tcp comment Dashboard: prometheus
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562160 ACTION=passthru ARGS=ufw allow from 10.100.60.2 to any port 9093 proto tcp comment Dashboard: alertmanager
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562168 ACTION=passthru ARGS=ufw allow from 10.100.60.2 to any port 8123 proto tcp comment Dashboard: clickhouse
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562176 ACTION=passthru ARGS=ufw allow from 10.100.60.2 to any port 3000 proto tcp comment Dashboard: grafana
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562184 ACTION=passthru ARGS=ufw allow from 10.100.60.2 to any port 4318 proto tcp comment Dashboard: otelcol
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562192 ACTION=passthru ARGS=ufw allow from 10.100.60.2 to any port 3100 proto tcp comment Dashboard: loki
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:56 UTC] USER=www-data EUID=0 PID=1562200 ACTION=passthru ARGS=ufw allow from 10.100.60.2 to any port 3200 proto tcp comment Dashboard: tempo
ERROR: passthru not allowed: ufw
[0;34m[INFO][0m Allowing dashboard access from 142.93.238.16...
[2026-01-02 06:41:57 UTC] USER=www-data EUID=0 PID=1562208 ACTION=passthru ARGS=ufw allow from 142.93.238.16 to any port 9090 proto tcp comment Dashboard: prometheus
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:57 UTC] USER=www-data EUID=0 PID=1562216 ACTION=passthru ARGS=ufw allow from 142.93.238.16 to any port 9093 proto tcp comment Dashboard: alertmanager
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:57 UTC] USER=www-data EUID=0 PID=1562224 ACTION=passthru ARGS=ufw allow from 142.93.238.16 to any port 8123 proto tcp comment Dashboard: clickhouse
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:57 UTC] USER=www-data EUID=0 PID=1562236 ACTION=passthru ARGS=ufw allow from 142.93.238.16 to any port 3000 proto tcp comment Dashboard: grafana
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:57 UTC] USER=www-data EUID=0 PID=1562244 ACTION=passthru ARGS=ufw allow from 142.93.238.16 to any port 4318 proto tcp comment Dashboard: otelcol
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:57 UTC] USER=www-data EUID=0 PID=1562252 ACTION=passthru ARGS=ufw allow from 142.93.238.16 to any port 3100 proto tcp comment Dashboard: loki
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:57 UTC] USER=www-data EUID=0 PID=1562260 ACTION=passthru ARGS=ufw allow from 142.93.238.16 to any port 3200 proto tcp comment Dashboard: tempo
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:57 UTC] USER=www-data EUID=0 PID=1562268 ACTION=passthru ARGS=ufw allow 443/tcp comment HTTPS obs-proxy
ERROR: passthru not allowed: ufw
[2026-01-02 06:41:57 UTC] USER=www-data EUID=0 PID=1562276 ACTION=passthru ARGS=ufw reload
ERROR: passthru not allowed: ufw
[0;32m[OK][0m UFW firewall rules configured
[0;32m[OK][0m ═══════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ Firewall configuration completed
[0;32m[OK][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Current firewall status:
[0;32m[OK][0m Firewall rules configured
[0;34m[INFO][0m Step 13/13: Configuring OAuth/SSO...
[0;34m[INFO][0m OAuth/SSO configuration script not found, skipping...
[0;34m[INFO][0m Running validation checks...
[0;34m[INFO][0m Validation script not found, skipping...
[0;34m[INFO][0m Registering observability components to dashboard...
[0;34m[INFO][0m Components to register: metrics alerts dashboards traces telemetry logstore proxy
[0;34m[INFO][0m Skipping metrics - registered by deploy script
[0;34m[INFO][0m Skipping alerts - registered by deploy script
[0;34m[INFO][0m Skipping dashboards - registered by deploy script
[0;34m[INFO][0m Skipping traces - registered by deploy script
[0;34m[INFO][0m Skipping telemetry - registered by deploy script
[0;34m[INFO][0m Skipping logstore - registered by deploy script
[0;34m[INFO][0m Processing component: proxy
[0;34m[INFO][0m Registering: proxy (obs-identity-sau-main-dev-proxy)
[INFO] Detected observability component, parsing: identity-sau-main-dev-proxy
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: Observability Proxy
[INFO] Identifier: obs-identity-sau-main-dev-proxy
[INFO] Identifier Parent: observability-cell
[INFO] IP: 10.100.1.51
[INFO] Port: 443
[INFO] FQDN: observe-identity-sau-main-dev.fastorder.com
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: 97579a1b-98ae-48e8-b639-bfdee43bca78
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[0;32m[OK][0m ✓ Registered: proxy
[0;34m[INFO][0m Registering short DNS aliases...
[0;32m[OK][0m ✓ Observability components registration completed
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════════════════════
[0;34m[INFO][0m Verifying all observability services are running...
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════════════════════
[0;32m[OK][0m ✓ grafana-obs-identity-sau-main-dev.service is running
[0;32m[OK][0m ✓ prometheus-obs-identity-sau-main-dev.service is running
[0;32m[OK][0m ✓ tempo-obs-identity-sau-main-dev.service is running
[0;32m[OK][0m ✓ All observability services verified running
═══════════════════════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ OBSERVABILITY CELL PROVISIONED: obs-identity-sau-main-dev
═══════════════════════════════════════════════════════════════════════════════
[0;34m[INFO][0m DNS Entries:
metrics-identity-sau-main-dev-prometheus.fastorder.com (10.100.1.205)
alerts-identity-sau-main-dev-alertmanager.fastorder.com (10.100.1.209)
dashboards-identity-sau-main-dev-grafana.fastorder.com (10.100.1.206)
traces-identity-sau-main-dev-tempo.fastorder.com (10.100.1.208)
telemetry-identity-sau-main-dev-opentelemetry.fastorder.com (10.100.1.210)
logstore-identity-sau-main-dev-clickhouse.fastorder.com (10.100.1.207)
observe-identity-sau-main-dev.fastorder.com (10.100.1.51)
[0;34m[INFO][0m Secrets Path: fastorder/observability/identity/sau/dev/*
[0;34m[INFO][0m Access (Purpose-Oriented URLs):
Dashboards: https://dashboards-identity-sau-main-dev-grafana.fastorder.com (SSO enabled)
Metrics: https://metrics-identity-sau-main-dev-prometheus.fastorder.com (internal only)
Alerts: https://alerts-identity-sau-main-dev-alertmanager.fastorder.com
Log Storage: https://logstore-identity-sau-main-dev-clickhouse.fastorder.com
[0;34m[INFO][0m Backend Implementation (Internal - Not Exposed to Clients):
Telemetry: otlp
Metrics: prometheus
Traces: tempo
Dashboards: grafana
Alerting: alertmanager
Log Storage: clickhouse
[0;34m[INFO][0m For applications in identity-sau-main-dev:
- Metrics: Push to telemetry-identity-sau-main-dev-opentelemetry.fastorder.com:4318 (OTLP/HTTP)
- Logs: Push to telemetry-identity-sau-main-dev-opentelemetry.fastorder.com:4318 (OTLP/HTTP)
- Traces: Push to telemetry-identity-sau-main-dev-opentelemetry.fastorder.com:4317 (OTLP/gRPC)
- Query Metrics: https://metrics-identity-sau-main-dev-prometheus.fastorder.com
- Query Logs: https://logstore-identity-sau-main-dev-clickhouse.fastorder.com
- Query Traces: https://traces-identity-sau-main-dev-tempo.fastorder.com
[0;34m[INFO][0m Runbook: /opt/fastorder/bash/scripts/env_app_setup/setup/02-observability-cell/RUNBOOK.md
═══════════════════════════════════════════════════════════════════════════════
[0;34m[INFO][0m Using search engine from SEARCH_ENGINE environment variable: elasticsearch
[0;34m[INFO][0m Cleaning up any existing locks...
[0;32m[1mStarting search engine: elasticsearch[0m
[1;33m═══════════════════════════════════════════════[0m
[0;36m[1m════════════════════════════════════════════════════════════════[0m
[0;36m[1m Elasticsearch Deployment Runner [0m
[0;36m[1m════════════════════════════════════════════════════════════════[0m
[0;34m[INFO][0m Cleaning up any existing locks (without triggering package configurations)...
[1;33m[WARNING][0m Lock cleanup skipped (wrapper not available or insufficient permissions)
[0;32m[1m🚀 Auto mode enabled - running automatic installation[0m
[0;32m[1mStarting Automatic Installation...[0m
[1;33m═══════════════════════════════════════════════[0m
[0;34mWill execute all deployment tasks in sequence:[0m
[0;32m[1m[1][0m Install Elasticsearch Http [0;35m(01-install-elasticsearch-http)[0m
[0;32m[1m[2][0m Make Https [0;35m(02-make-https)[0m
[0;32m[1m[3][0m Create Index Llm [0;35m(03-create-index-llm)[0m
[0;32m[1m[4][0m Monitoring Setup [0;35m(10-monitoring-setup)[0m
[1;33m═══════════════════════════════════════════════[0m
[0;32m🚀 Auto mode - proceeding automatically...[0m
[0;32m[1mRunning automatic installation...[0m
[0;36m[1m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;32m[1mStep 1: Executing Install Elasticsearch Http[0m
[0;35mFolder: 01-install-elasticsearch-http[0m
[0;36m[1m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
=== Elasticsearch HTTP Setup ===
Install and configure Elasticsearch with HTTP access
Architecture: Per-node VM IPs with default port (9200)
[INFO] Using web-provided environment: identity-sau-main-dev
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment:
Nodes: 1
Port: 9200 (default Elasticsearch port)
Coordinator endpoint: http://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
Checking if Elasticsearch is already installed for environment: ...
Validating Elasticsearch installation...
./run.sh: line 132: /var/www/html/skeleton.dev.fastorder.com/fixing/scripts/lib/elasticsearch_validator.sh: No such file or directory
⚠️ Elasticsearch installation issues detected. Attempting automatic repair...
./run.sh: line 134: /var/www/html/skeleton.dev.fastorder.com/fixing/scripts/lib/elasticsearch_validator.sh: No such file or directory
Executing: steps/01-setup-directories.sh
+ 01-setup-directories.sh:4:main: echo '=== Step 1: Creating directory structure ==='
=== Step 1: Creating directory structure ===
+++ 01-setup-directories.sh:4:main: dirname steps/01-setup-directories.sh
++ 01-setup-directories.sh:4:main: cd steps
++ 01-setup-directories.sh:4:main: pwd
+ 01-setup-directories.sh:4:main: SCRIPT_DIR=/opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/01-install-elasticsearch-http/steps
+ 01-setup-directories.sh:4:main: source /opt/fastorder/bash/scripts/env_app_setup/lib/provisioning-init.sh
++ 01-setup-directories.sh:4:main: RED='\033[0;31m'
++ 01-setup-directories.sh:4:main: GREEN='\033[0;32m'
++ 01-setup-directories.sh:4:main: YELLOW='\033[1;33m'
++ 01-setup-directories.sh:4:main: BLUE='\033[0;34m'
++ 01-setup-directories.sh:4:main: NC='\033[0m'
++ 01-setup-directories.sh:4:main: export TERM=dumb
++ 01-setup-directories.sh:4:main: TERM=dumb
++ 01-setup-directories.sh:4:main: export DEBIAN_FRONTEND=noninteractive
++ 01-setup-directories.sh:4:main: DEBIAN_FRONTEND=noninteractive
++ 01-setup-directories.sh:4:main: export NEEDRESTART_MODE=a
++ 01-setup-directories.sh:4:main: NEEDRESTART_MODE=a
++ 01-setup-directories.sh:4:main: export NEEDRESTART_SUSPEND=1
++ 01-setup-directories.sh:4:main: NEEDRESTART_SUSPEND=1
++ 01-setup-directories.sh:4:main: export DEBIAN_PRIORITY=critical
++ 01-setup-directories.sh:4:main: DEBIAN_PRIORITY=critical
++ 01-setup-directories.sh:4:main: export UCF_FORCE_CONFFOLD=1
++ 01-setup-directories.sh:4:main: UCF_FORCE_CONFFOLD=1
++ 01-setup-directories.sh:4:main: export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
++ 01-setup-directories.sh:4:main: PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
++ 01-setup-directories.sh:4:main: [[ -n '' ]]
++ 01-setup-directories.sh:4:main: [[ -n /opt/fastorder/bash/scripts/env_app_setup/state ]]
++ 01-setup-directories.sh:4:main: [[ -d /opt/fastorder/bash/scripts/env_app_setup/state ]]
++ 01-setup-directories.sh:4:main: STATE_DIR=/opt/fastorder/bash/scripts/env_app_setup/state
++ 01-setup-directories.sh:4:main: export STATE_DIR
++ 01-setup-directories.sh:4:main: [[ -f /opt/fastorder/bash/scripts/env_app_setup/setup/setup.json ]]
++ 01-setup-directories.sh:4:main: SETUP_JSON=/opt/fastorder/bash/scripts/env_app_setup/setup/setup.json
++ 01-setup-directories.sh:4:main: FO_WRAPPER=/usr/local/bin/fastorder-provisioning-wrapper.sh
++ 01-setup-directories.sh:4:main: HTTP_PORT_BASE=9200
++ 01-setup-directories.sh:4:main: TRANSPORT_PORT_BASE=9300
++ 01-setup-directories.sh:4:main: PG_PORT_BASE=5432
++ 01-setup-directories.sh:4:main: APP_IP_SUBNETS=(['observability']='10.100.5' ['obs']='10.100.5' ['prometheus']='10.100.5' ['grafana']='10.100.5' ['loki']='10.100.5' ['tempo']='10.100.5' ['postgresql']='10.100.10' ['postgres']='10.100.10' ['pg']='10.100.10' ['elasticsearch']='10.100.20' ['es']='10.100.20' ['kafka']='10.100.30' ['redis']='10.100.40' ['mongodb']='10.100.50' ['mongo']='10.100.50' ['iam']='10.100.60' ['keycloak']='10.100.60' ['general']='10.100.1')
++ 01-setup-directories.sh:4:main: declare -A APP_IP_SUBNETS
++ 01-setup-directories.sh:4:main: APP_IP_RESERVED_START=(['observability']='2' ['postgresql']='2' ['elasticsearch']='2' ['kafka']='2' ['redis']='2' ['mongodb']='2' ['iam']='2' ['general']='50')
++ 01-setup-directories.sh:4:main: declare -A APP_IP_RESERVED_START
++ 01-setup-directories.sh:4:main: APP_IP_RESERVED_END=(['observability']='49' ['postgresql']='254' ['elasticsearch']='254' ['kafka']='254' ['redis']='254' ['mongodb']='254' ['iam']='254' ['general']='250')
++ 01-setup-directories.sh:4:main: declare -A APP_IP_RESERVED_END
+++ 01-setup-directories.sh:4:main: dirname /opt/fastorder/bash/scripts/env_app_setup/lib/provisioning-init.sh
++ 01-setup-directories.sh:4:main: _CONFIG_MGMT_LIB=/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh
++ 01-setup-directories.sh:4:main: [[ -f /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh ]]
++ 01-setup-directories.sh:4:main: source /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh
+++ 01-setup-directories.sh:4:main: set -Eeuo pipefail
+++ 01-setup-directories.sh:4:main: : /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator
+++ 01-setup-directories.sh:4:main: STATE_DIR=/opt/fastorder/bash/scripts/env_app_setup/state
++ 01-setup-directories.sh:4:main: [[ /opt/fastorder/bash/scripts/env_app_setup/lib/provisioning-init.sh == \s\t\e\p\s\/\0\1\-\s\e\t\u\p\-\d\i\r\e\c\t\o\r\i\e\s\.\s\h ]]
++ 01-setup-directories.sh:4:main: set +e
++ 01-setup-directories.sh:4:main: set +u
++ 01-setup-directories.sh:4:main: set +o pipefail
++ 01-setup-directories.sh:4:main: set +E
+ 01-setup-directories.sh:4:main: source /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/01-install-elasticsearch-http/steps/lib/setup_directories_per_node.sh
++ 01-setup-directories.sh:4:main: [[ /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/01-install-elasticsearch-http/steps/lib/setup_directories_per_node.sh == \s\t\e\p\s\/\0\1\-\s\e\t\u\p\-\d\i\r\e\c\t\o\r\i\e\s\.\s\h ]]
+ 01-setup-directories.sh:4:main: init_environment
+ 01-setup-directories.sh:4:main: require_bin jq
+ 01-setup-directories.sh:4:main: for b in "$@"
+ 01-setup-directories.sh:4:main: command -v jq
+ 01-setup-directories.sh:4:main: local app_type=general
+ 01-setup-directories.sh:4:main: ENV_ID=identity-sau-main-dev
+ 01-setup-directories.sh:4:main: [[ -z identity-sau-main-dev ]]
+ 01-setup-directories.sh:4:main: [[ -z identity-sau-main-dev ]]
+ 01-setup-directories.sh:4:main: ENV_ID=identity-sau-main-dev
+ 01-setup-directories.sh:4:main: [[ -z identity-sau-main-dev ]]
+ 01-setup-directories.sh:4:main: [[ -z identity-sau-main-dev ]]
+ 01-setup-directories.sh:4:main: [[ -z identity-sau-main-dev ]]
++ 01-setup-directories.sh:4:main: env_dir_for identity-sau-main-dev
++ 01-setup-directories.sh:4:main: echo /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev
+ 01-setup-directories.sh:4:main: ENV_DIR=/opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev
++ 01-setup-directories.sh:4:main: topo_path_for identity-sau-main-dev
++ 01-setup-directories.sh:4:main: echo /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: TOPOLOGY_JSON=/opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: [[ ! -f /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json ]]
+ 01-setup-directories.sh:4:main: validate_topology_json /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: local topo=/opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: [[ -r /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json ]]
+ 01-setup-directories.sh:4:main: jq -e '
.schema_version == 1
and (.general.id | type=="string")
and (.general.shared_ip | type=="string")
and (.general.service | type=="string")
and (.general.zone | type=="string")
and (.general.env | type=="string")
' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
++ 01-setup-directories.sh:4:main: jq -r .general.service /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: SERVICE=identity
++ 01-setup-directories.sh:4:main: jq -r .general.zone /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: zone=sau
++ 01-setup-directories.sh:4:main: jq -r .general.branch /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: BRANCH=main
++ 01-setup-directories.sh:4:main: jq -r .general.env /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: ENV=dev
++ 01-setup-directories.sh:4:main: jq -r '.general.es_nodes_num // 3' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: ES_NODES_NUM=1
++ 01-setup-directories.sh:4:main: jq -r '.general.pg_workers_num // 3' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: PG_WORKERS_NUM=1
++ 01-setup-directories.sh:4:main: jq -r '.general.pg_WORKERS_STANDBY_NUM // 3' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: PG_WORKERS_STANDBY_NUM=3
++ 01-setup-directories.sh:4:main: jq -r '.general.pg_citus_enabled // "yes"' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: PG_CITUS_ENABLED=yes
+ 01-setup-directories.sh:4:main: [[ general != \g\e\n\e\r\a\l ]]
++ 01-setup-directories.sh:4:main: jq -r .general.shared_ip /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: VM_IP=142.93.238.16
+ 01-setup-directories.sh:4:main: [[ general != \g\e\n\e\r\a\l ]]
++ 01-setup-directories.sh:4:main: jq -r '.general.shared_iface // empty' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: IFACE=eth0:16
+ 01-setup-directories.sh:4:main: local FINAL_VM_IP=142.93.238.16
+ 01-setup-directories.sh:4:main: set -a
+ 01-setup-directories.sh:4:main: [[ -r /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/generated/general.env ]]
+ 01-setup-directories.sh:4:main: source /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/generated/general.env
++ 01-setup-directories.sh:4:main: ENV_ID=identity-sau-main-dev
++ 01-setup-directories.sh:4:main: SERVICE=identity
++ 01-setup-directories.sh:4:main: zone=sau
++ 01-setup-directories.sh:4:main: BRANCH=main
++ 01-setup-directories.sh:4:main: ENV=dev
++ 01-setup-directories.sh:4:main: VM_IP=142.93.238.16
++ 01-setup-directories.sh:4:main: IFACE=eth0:16
++ 01-setup-directories.sh:4:main: ROOT_DIR=/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator
++ 01-setup-directories.sh:4:main: ENV_DIR=/opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev
++ 01-setup-directories.sh:4:main: TOPOLOGY_JSON=/opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
++ 01-setup-directories.sh:4:main: LOG_LEVEL=info
++ 01-setup-directories.sh:4:main: DEBUG_MODE=false
+ 01-setup-directories.sh:4:main: set +a
+ 01-setup-directories.sh:4:main: VM_IP=142.93.238.16
+ 01-setup-directories.sh:4:main: export ENV_ID SERVICE zone BRANCH ENV VM_IP IFACE ENV_DIR TOPOLOGY_JSON
+ 01-setup-directories.sh:4:main: export ES_NODES_NUM PG_WORKERS_NUM PG_WORKERS_STANDBY_NUM PG_CITUS_ENABLED
+ 01-setup-directories.sh:4:main: [[ general != \g\e\n\e\r\a\l ]]
+ 01-setup-directories.sh:4:main: info 'Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)'
+ 01-setup-directories.sh:4:main: printf '[INFO] %s\n' 'Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)'
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
+ 01-setup-directories.sh:4:main: return 0
+ 01-setup-directories.sh:4:main: SERVICE=identity
+ 01-setup-directories.sh:4:main: ZONE=sau
+ 01-setup-directories.sh:4:main: BRANCH=main
+ 01-setup-directories.sh:4:main: ENV=dev
++ 01-setup-directories.sh:4:main: env_id
++ 01-setup-directories.sh:4:main: '[' identity = auth ']'
++ 01-setup-directories.sh:4:main: '[' identity = item ']'
++ 01-setup-directories.sh:4:main: echo identity-sau-main-dev
+ 01-setup-directories.sh:4:main: ENV_ID=identity-sau-main-dev
+ 01-setup-directories.sh:4:main: env=identity-sau-main-dev
+ 01-setup-directories.sh:4:main: nodes=1
+ 01-setup-directories.sh:4:main: [[ 1 =~ ^[1-9][0-9]*$ ]]
+ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop mkdir -p /etc/elasticsearch
[2026-01-02 06:42:19 UTC] USER=www-data EUID=0 PID=1562753 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch
+ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop mkdir -p /var/lib/elasticsearch
[2026-01-02 06:42:19 UTC] USER=www-data EUID=0 PID=1562762 ACTION=fsop ARGS=mkdir -p /var/lib/elasticsearch
+ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop mkdir -p /var/log/elasticsearch
[2026-01-02 06:42:19 UTC] USER=www-data EUID=0 PID=1562771 ACTION=fsop ARGS=mkdir -p /var/log/elasticsearch
+ 01-setup-directories.sh:4:main: APP_NAME=search
+ 01-setup-directories.sh:4:main: TOPOLOGY_FILE=/opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: command -v jq
+ 01-setup-directories.sh:4:main: [[ -f /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json ]]
++ 01-setup-directories.sh:4:main: jq -r --arg app search '.applications[$app].vm_ip // empty' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: COORD_IP=10.100.1.94
+ 01-setup-directories.sh:4:main: [[ -z 10.100.1.94 ]]
+ 01-setup-directories.sh:4:main: [[ 10.100.1.94 == \n\u\l\l ]]
++ 01-setup-directories.sh:4:main: get_application_domain search
++ 01-setup-directories.sh:4:main: local app_type=search
++ 01-setup-directories.sh:4:main: [[ search == \g\e\n\e\r\a\l ]]
++ 01-setup-directories.sh:4:main: jq -r --arg app search '.applications[$app].domain // empty' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
+ 01-setup-directories.sh:4:main: COORD_DOMAIN=search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com
+ 01-setup-directories.sh:4:main: info 'Coordinator exists: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com (10.100.1.94)'
+ 01-setup-directories.sh:4:main: printf '[INFO] %s\n' 'Coordinator exists: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com (10.100.1.94)'
[INFO] Coordinator exists: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com (10.100.1.94)
+ 01-setup-directories.sh:4:main: (( i=1 ))
+ 01-setup-directories.sh:4:main: (( i<=nodes ))
++ 01-setup-directories.sh:4:main: printf %02d 1
+ 01-setup-directories.sh:4:main: node_num=01
+ 01-setup-directories.sh:4:main: IDENTIFIER=node-01
+ 01-setup-directories.sh:4:main: APP_NAME=search-node-01
+ 01-setup-directories.sh:4:main: read -r NODE_IP NODE_DOMAIN
++ 01-setup-directories.sh:4:main: setup_directories_per_node node-01 search-node-01
++ 01-setup-directories.sh:4:main: local IDENTIFIER=node-01
++ 01-setup-directories.sh:4:main: local APP_NAME=search-node-01
++ 01-setup-directories.sh:4:main: local env
+++ 01-setup-directories.sh:4:main: env_id
+++ 01-setup-directories.sh:4:main: '[' identity = auth ']'
+++ 01-setup-directories.sh:4:main: '[' identity = item ']'
+++ 01-setup-directories.sh:4:main: echo identity-sau-main-dev
++ 01-setup-directories.sh:4:main: env=identity-sau-main-dev
++ 01-setup-directories.sh:4:main: local TOPOLOGY_FILE=/opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
++ 01-setup-directories.sh:4:main: info 'Setting up Elasticsearch node: node-01'
++ 01-setup-directories.sh:4:main: printf '[INFO] %s\n' 'Setting up Elasticsearch node: node-01'
++ 01-setup-directories.sh:4:main: local NODE_IP NODE_DOMAIN
+ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop ln -sfn /etc/elasticsearch/identity-sau-main-dev/node-01 /etc/elasticsearch/identity-sau-main-dev-node-01
+++ 01-setup-directories.sh:4:main: jq -r --arg app search-node-01 '.applications[$app].vm_ip // empty' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[2026-01-02 06:42:20 UTC] USER=www-data EUID=0 PID=1562789 ACTION=fsop ARGS=ln -sfn /etc/elasticsearch/identity-sau-main-dev/node-01 /etc/elasticsearch/identity-sau-main-dev-node-01
+ 01-setup-directories.sh:4:main: [[ 1 -eq 1 ]]
+ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop ln -sfn /etc/elasticsearch/identity-sau-main-dev/node-01 /etc/elasticsearch/current
++ 01-setup-directories.sh:4:main: NODE_IP=10.100.1.103
++ 01-setup-directories.sh:4:main: [[ -z 10.100.1.103 ]]
++ 01-setup-directories.sh:4:main: [[ 10.100.1.103 == \n\u\l\l ]]
+++ 01-setup-directories.sh:4:main: get_application_domain search-node-01
+++ 01-setup-directories.sh:4:main: local app_type=search-node-01
+++ 01-setup-directories.sh:4:main: [[ search-node-01 == \g\e\n\e\r\a\l ]]
+++ 01-setup-directories.sh:4:main: jq -r --arg app search-node-01 '.applications[$app].domain // empty' /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
[2026-01-02 06:42:20 UTC] USER=www-data EUID=0 PID=1562800 ACTION=fsop ARGS=ln -sfn /etc/elasticsearch/identity-sau-main-dev/node-01 /etc/elasticsearch/current
+ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop ln -sfn /var/lib/elasticsearch/identity-sau-main-dev/node-01 /var/lib/elasticsearch/current
++ 01-setup-directories.sh:4:main: NODE_DOMAIN=search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com
++ 01-setup-directories.sh:4:main: info 'Using existing node-01: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)'
++ 01-setup-directories.sh:4:main: printf '[INFO] %s\n' 'Using existing node-01: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)'
/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh: line 13: printf: write error: Broken pipe
++ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01/certs
[2026-01-02 06:42:20 UTC] USER=www-data EUID=0 PID=1562809 ACTION=fsop ARGS=ln -sfn /var/lib/elasticsearch/identity-sau-main-dev/node-01 /var/lib/elasticsearch/current
+ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop ln -sfn /var/log/elasticsearch/identity-sau-main-dev/node-01 /var/log/elasticsearch/current
+ 01-setup-directories.sh:4:main: (( i++ ))
+ 01-setup-directories.sh:4:main: (( i<=nodes ))
+ 01-setup-directories.sh:4:main: success 'Directory structure created for '\''identity-sau-main-dev'\'' with 1 node(s).'
+ 01-setup-directories.sh:4:main: printf '[ OK ] %s\n' 'Directory structure created for '\''identity-sau-main-dev'\'' with 1 node(s).'
[ OK ] Directory structure created for 'identity-sau-main-dev' with 1 node(s).
Executing: steps/02-install-dependencies.sh
++ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop mkdir -p /var/lib/elasticsearch/identity-sau-main-dev/node-01/tmp
=== Step 2: Installing/Validating Elasticsearch (latest) ===
++ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop mkdir -p /var/log/elasticsearch/identity-sau-main-dev/node-01
++ 01-setup-directories.sh:4:main: id -u elasticsearch
++ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01
++ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop chown -R elasticsearch:elasticsearch /var/lib/elasticsearch/identity-sau-main-dev/node-01
++ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop chown -R elasticsearch:elasticsearch /var/log/elasticsearch/identity-sau-main-dev/node-01
++ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop chmod 0750 /etc/elasticsearch/identity-sau-main-dev/node-01
++ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop chmod 0750 /var/lib/elasticsearch/identity-sau-main-dev/node-01
++ 01-setup-directories.sh:4:main: command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh fsop chmod 0750 /var/log/elasticsearch/identity-sau-main-dev/node-01
++ 01-setup-directories.sh:4:main: info 'Created dirs for identity-sau-main-dev/node-01 @ 10.100.1.103'
++ 01-setup-directories.sh:4:main: printf '[INFO] %s\n' 'Created dirs for identity-sau-main-dev/node-01 @ 10.100.1.103'
/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh: line 13: printf: write error: Broken pipe
++ 01-setup-directories.sh:4:main: printf '%s\n' 10.100.1.103
/opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/01-install-elasticsearch-http/steps/lib/setup_directories_per_node.sh: line 58: printf: write error: Broken pipe
++ 01-setup-directories.sh:4:main: printf '%s\n' search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com
/opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/01-install-elasticsearch-http/steps/lib/setup_directories_per_node.sh: line 59: printf: write error: Broken pipe
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] Cleaning dpkg/apt locks...
[2026-01-02 06:42:21 UTC] USER=www-data EUID=0 PID=1562939 ACTION=cleanup-dpkg-locks ARGS=
steps/02-install-dependencies.sh: line 16: 1562937 Killed command sudo -n "$WRAP" cleanup-dpkg-locks
[2026-01-02 06:42:21 UTC] USER=www-data EUID=0 PID=1562948 ACTION=fsop ARGS=mkdir -p /etc/apt/keyrings
[2026-01-02 06:42:21 UTC] USER=www-data EUID=0 PID=1562957 ACTION=fsop ARGS=chmod 0755 /etc/apt/keyrings
[INFO] apt-get update…
[2026-01-02 06:42:21 UTC] USER=www-data EUID=0 PID=1562967 ACTION=pkg ARGS=update
Hit:1 http://apt.postgresql.org/pub/repos/apt jammy-pgdg InRelease
Hit:2 https://artifacts.elastic.co/packages/8.x/apt stable InRelease
Hit:3 https://packages.confluent.io/deb/7.6 stable InRelease
Hit:4 https://apt.grafana.com stable InRelease
Hit:5 https://deb.nodesource.com/node_22.x nodistro InRelease
Hit:6 https://mirrors.edge.kernel.org/ubuntu jammy InRelease
Hit:7 https://mirrors.edge.kernel.org/ubuntu jammy-backports InRelease
Hit:8 https://mirrors.edge.kernel.org/ubuntu jammy-security InRelease
Hit:9 https://mirrors.edge.kernel.org/ubuntu jammy-updates InRelease
Hit:10 https://packages.microsoft.com/repos/azure-cli jammy InRelease
Hit:11 https://packages.clickhouse.com/deb stable InRelease
Hit:12 https://ppa.launchpadcontent.net/ondrej/php/ubuntu jammy InRelease
Hit:13 https://repos.citusdata.com/community/ubuntu jammy InRelease
Reading package lists...
[INFO] Installed version : 8.19.9
[INFO] Candidate version : 8.19.9
✅ Elasticsearch already at latest (or only) available version.
✅ Elasticsearch installation validated.
🎉 Dependencies installed and up-to-date.
Executing: steps/03-create-env-configs.sh
=== Step 3: Creating environment configurations (master + nodes, TLS, units) ===
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Configuring env: identity-sau-main-dev (nodes: 1, http: 9200, transport: 9300)
Using heap size: 1024m per node
[2026-01-02 06:42:52 UTC] USER=www-data EUID=0 PID=1564409 ACTION=fsop ARGS=chown root:root /etc/default/elasticsearch
[2026-01-02 06:42:52 UTC] USER=www-data EUID=0 PID=1564418 ACTION=fsop ARGS=chmod 0644 /etc/default/elasticsearch
[2026-01-02 06:42:52 UTC] USER=www-data EUID=0 PID=1564436 ACTION=fsop ARGS=chown elasticsearch:elasticsearch /etc/elasticsearch/jvm.options
[2026-01-02 06:42:53 UTC] USER=www-data EUID=0 PID=1564445 ACTION=fsop ARGS=chmod 0644 /etc/elasticsearch/jvm.options
[2026-01-02 06:42:53 UTC] USER=www-data EUID=0 PID=1564463 ACTION=fsop ARGS=mkdir -p /etc/systemd/system.conf.d /etc/systemd/user.conf.d
[2026-01-02 06:42:53 UTC] USER=www-data EUID=0 PID=1564490 ACTION=passthru ARGS=systemctl daemon-reload
Current max_map_count: 262144
Current swappiness: 1
[2026-01-02 06:42:54 UTC] USER=www-data EUID=0 PID=1564581 ACTION=fsop ARGS=chown elasticsearch:elasticsearch /etc/elasticsearch/log4j2.properties
[2026-01-02 06:42:54 UTC] USER=www-data EUID=0 PID=1564591 ACTION=fsop ARGS=chmod 0644 /etc/elasticsearch/log4j2.properties
[2026-01-02 06:42:54 UTC] USER=www-data EUID=0 PID=1564600 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/template
[2026-01-02 06:42:54 UTC] USER=www-data EUID=0 PID=1564609 ACTION=fsop ARGS=chown elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev /etc/elasticsearch/identity-sau-main-dev/template
[2026-01-02 06:42:54 UTC] USER=www-data EUID=0 PID=1564618 ACTION=fsop ARGS=chmod 0755 /etc/elasticsearch/identity-sau-main-dev
[2026-01-02 06:42:54 UTC] USER=www-data EUID=0 PID=1564627 ACTION=fsop ARGS=cp /etc/elasticsearch/jvm.options /etc/elasticsearch/identity-sau-main-dev/template/jvm.options
[INFO] 🌐 Registering general environment domain: identity-sau-main-dev.fastorder.com
[INFO] Allocated VM IP: 10.100.1.50 for general environment
[INFO] Configuring VM IP 10.100.1.50 on network interface...
[1;33m[WARNING][0m VM IP may already be configured or need manual setup
[1;33m[WARNING][0m Warning: VM IP 10.100.1.50 not found on network interfaces
[ OK ] ✅ Registered general domain identity-sau-main-dev.fastorder.com -> 10.100.1.50
[ OK ] ✅ DNS resolution verified for identity-sau-main-dev.fastorder.com
[INFO] → Configuring identity-sau-main-dev-node-01 (10.100.1.103) roles=[ master, data, data_hot, data_content, ingest ]
[2026-01-02 06:42:55 UTC] USER=www-data EUID=0 PID=1564718 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01/certs /var/lib/elasticsearch/identity-sau-main-dev/node-01/tmp /var/log/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:42:55 UTC] USER=www-data EUID=0 PID=1564727 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:42:55 UTC] USER=www-data EUID=0 PID=1564742 ACTION=fsop ARGS=chmod 0750 /etc/elasticsearch/identity-sau-main-dev/node-01 /var/lib/elasticsearch/identity-sau-main-dev/node-01 /var/log/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:42:55 UTC] USER=www-data EUID=0 PID=1564751 ACTION=fsop ARGS=cp /etc/elasticsearch/identity-sau-main-dev/template/jvm.options /etc/elasticsearch/identity-sau-main-dev/node-01/jvm.options
[2026-01-02 06:42:55 UTC] USER=www-data EUID=0 PID=1564760 ACTION=fsop ARGS=sed -i s/^-Xms.*/-Xms1024m/ /etc/elasticsearch/identity-sau-main-dev/node-01/jvm.options
[2026-01-02 06:42:55 UTC] USER=www-data EUID=0 PID=1564769 ACTION=fsop ARGS=sed -i s/^-Xmx.*/-Xmx1024m/ /etc/elasticsearch/identity-sau-main-dev/node-01/jvm.options
[2026-01-02 06:42:55 UTC] USER=www-data EUID=0 PID=1564788 ACTION=fsop ARGS=cp /etc/elasticsearch/log4j2.properties /etc/elasticsearch/identity-sau-main-dev/node-01/log4j2.properties
[2026-01-02 06:42:55 UTC] USER=www-data EUID=0 PID=1564826 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:42:56 UTC] USER=www-data EUID=0 PID=1564836 ACTION=fsop ARGS=chmod 0644 /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.yml
[2026-01-02 06:42:56 UTC] USER=www-data EUID=0 PID=1564858 ACTION=fsop ARGS=chmod 0644 /etc/default/elasticsearch-identity-sau-main-dev-node-01
[2026-01-02 06:42:56 UTC] USER=www-data EUID=0 PID=1564881 ACTION=passthru ARGS=ip addr add 10.100.1.103/32 dev eth0 label eth0:103
[2026-01-02 06:42:56 UTC] USER=www-data EUID=0 PID=1564890 ACTION=fsop ARGS=sed -i /[[:space:]]search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com\([[:space:]]\|$\)/d /etc/hosts
[INFO] → Also added short domain: search-identity-sau-main-dev.fastorder.com
[INFO] ✔ Created configuration for identity-sau-main-dev/node-01 (roles=single-node)
[2026-01-02 06:42:56 UTC] USER=www-data EUID=0 PID=1564920 ACTION=fsop ARGS=sed -i /[[:space:]]search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com\([[:space:]]\|$\)/d /etc/hosts
[INFO] ✔ Registered master domain search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com -> 10.100.1.103 (points to node-01)
[INFO] Cleaning up legacy non-templated elasticsearch-*.service units (if any)...
[INFO] No legacy units found.
[INFO] Base template exists: elasticsearch@.service
[ OK ] Created unit: elasticsearch@identity-sau-main-dev-node-01.service
[2026-01-02 06:42:56 UTC] USER=www-data EUID=0 PID=1564972 ACTION=passthru ARGS=systemctl daemon-reload
[ OK ] Environment configurations (master + nodes with TLS) created successfully!
[INFO] Environment: identity-sau-main-dev
[INFO] Nodes: 1
[INFO] HTTP Port: 9200
[INFO] Transport Port: 9300
[INFO] Heap Size: 1024m per node
[INFO] Master: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com (10.100.1.94)
[INFO] node-01: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
[INFO] Systemd units prepared (not started). Start sequence runs in Step 7.
Executing: steps/04-start-clusters.sh
=== Step 7: Starting Elasticsearch clusters (with waits) ===
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Starting Elasticsearch cluster for environment: identity-sau-main-dev (1 nodes)
[INFO] === Ensuring VM IP services are started ===
[1;33m[WARNING][0m VM IP service vm-ip-10-100-1-103.service not found - IP might not persist
[INFO] Manually configuring IP: 10.100.1.103
[2026-01-02 06:43:01 UTC] USER=www-data EUID=0 PID=1565105 ACTION=configure-network-interface ARGS=lo:search01 10.100.1.103
[INFO] Cleaning up any existing Elasticsearch processes and lock files...
[2026-01-02 06:43:01 UTC] USER=www-data EUID=0 PID=1565116 ACTION=passthru ARGS=systemctl is-active --quiet elasticsearch@identity-sau-main-dev-node-01.service
[INFO] Stopping Elasticsearch services for environment: identity-sau-main-dev ...
[INFO] No active Elasticsearch services found for environment: identity-sau-main-dev
[INFO] Removing lock files from: /var/lib/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:43:01 UTC] USER=www-data EUID=0 PID=1565129 ACTION=fsop ARGS=find /var/lib/elasticsearch/identity-sau-main-dev/node-01 -name *.lock -delete
[2026-01-02 06:43:02 UTC] USER=www-data EUID=0 PID=1565138 ACTION=fsop ARGS=find /var/lib/elasticsearch/identity-sau-main-dev/node-01 -name node.lock -delete
[2026-01-02 06:43:02 UTC] USER=www-data EUID=0 PID=1565147 ACTION=fsop ARGS=find /var/lib/elasticsearch/identity-sau-main-dev/node-01 -name _state -type d -exec rm -rf {} +
[2026-01-02 06:43:02 UTC] USER=www-data EUID=0 PID=1565157 ACTION=fsop ARGS=find /tmp -name *elasticsearch*identity-sau-main-dev-node-01* -delete
[ OK ] Cleanup completed for environment: identity-sau-main-dev
[INFO] Checking for port conflicts before starting Elasticsearch...
[INFO] Checking for port conflicts on 10.100.1.94:9200 and 10.100.1.94:9300...
[ OK ] ✓ Ports 9200 and 9300 are available on 10.100.1.94
[INFO] Ensuring correct ownership of Elasticsearch directories...
[2026-01-02 06:43:04 UTC] USER=www-data EUID=0 PID=1565216 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch
[2026-01-02 06:43:05 UTC] USER=www-data EUID=0 PID=1565232 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /var/lib/elasticsearch
[2026-01-02 06:43:08 UTC] USER=www-data EUID=0 PID=1565258 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /var/log/elasticsearch
[ OK ] Directory ownership fixed
[INFO] === Starting Elasticsearch Nodes ===
[INFO] Starting 1 node(s) for cluster
▶ Starting elasticsearch@identity-sau-main-dev-node-01.service (search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200)
[2026-01-02 06:43:08 UTC] USER=www-data EUID=0 PID=1565276 ACTION=passthru ARGS=systemctl is-enabled --quiet elasticsearch@identity-sau-main-dev-node-01.service
[2026-01-02 06:43:09 UTC] USER=www-data EUID=0 PID=1565337 ACTION=passthru ARGS=systemctl start elasticsearch@identity-sau-main-dev-node-01.service
⏳ Waiting for TCP search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200 to be accessible (timeout 360s)...
✅ Port 9200 is accessible on search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com.
⏳ Waiting for ES HTTP readiness on http://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200 (timeout 300s)...
[ OK ] ES HTTP ready on search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[ OK ] elasticsearch@identity-sau-main-dev-node-01.service is up and answering HTTP on search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[INFO] Node 1 started successfully
[INFO] Cluster with 1 node(s) started successfully
⏳ Waiting for the cluster to elect master and settle...
⏳ Waiting for cluster health=green via search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200 (timeout 300s)...
[ OK ] Cluster is GREEN (nodes="number_of_nodes") on search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
[ OK ] Cluster identity-sau-main-dev is healthy and green!
[INFO] === Final Status Check ===
[2026-01-02 06:44:17 UTC] USER=www-data EUID=0 PID=1566753 ACTION=passthru ARGS=systemctl is-active --quiet elasticsearch@identity-sau-main-dev-node-01.service
[ OK ] elasticsearch@identity-sau-main-dev-node-01.service is ACTIVE (search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200)
└── HTTP responding on search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200 ✓
[ OK ] All 1 node(s) in environment 'identity-sau-main-dev' are running successfully!
[INFO] Node endpoints:
- http://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[ OK ] Elasticsearch cluster started successfully!
[INFO] Environment: identity-sau-main-dev
[INFO] Nodes: 1
[INFO] Cluster endpoints:
- http://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[INFO] === Quick Cluster Information ===
Cluster Name: fastorder-identity-sau-main-dev
Node Name: identity-sau-main-dev-node-01
Version: 8.19.9
Architecture: 1 node(s), each on default port 9200
Cluster with 1 node(s) started successfully (each on port 9200)
Executing: steps/05-verify-setup.sh
=== Step 8: Verifying setup (with retries) ===
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Verifying environment: identity-sau-main-dev (1 nodes, Single-node)
Main HTTP endpoint: http://search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
Testing network connectivity to search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200...
✓ Domain connection available
Testing HTTP response...
[ OK ] ✓ identity-sau-main-dev is responding on search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
[INFO] === Cluster Health ===
{
"cluster_name" : "fastorder-identity-sau-main-dev",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 0,
"active_shards" : 0,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"unassigned_primary_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
[ OK ] Cluster status: GREEN ("number_of_nodes" nodes)
[INFO] === Cluster Nodes ===
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
10.100.1.103 53 98 68 5.76 5.58 5.25 dhims * identity-sau-main-dev-node-01
[INFO] === Single-Node Service Verification ===
Testing coordinator service (search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200)...
✓ Coordinator HTTP responding on search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
Name: identity-sau-main-dev-node-01, Version: 8.19.9
[INFO] === Cluster State Summary ===
Using jq for formatted output:
jq parsing failed
[ OK ] === Verification Summary ===
[INFO] Environment: identity-sau-main-dev
[INFO] Nodes configured: 1
[INFO] Main endpoint: http://search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
[INFO] Service endpoint: http://search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
[INFO] === Final Connectivity Test ===
✓ Coordinator: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
[ OK ] Single-node cluster is responding successfully!
[ OK ] Elasticsearch cluster 'identity-sau-main-dev' verification completed successfully!
Executing: steps/06-confirm-working.sh
=== Step 9: Comprehensive Cluster Verification (gated) ===
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
========================================
🔍 Verifying Environment: identity-sau-main-dev (1 nodes)
========================================
Domain: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com
Environment: identity-sau-main-dev
Nodes: 1
[INFO] Testing network connectivity...
Setup type: Single-node
Testing endpoint: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
[ OK ] ✓ Using domain: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com
📡 Coordinator Service (elasticsearch@identity-sau-main-dev-node-01.service)
Endpoint: search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
--------------------------------
[2026-01-02 06:44:20 UTC] USER=www-data EUID=0 PID=1566922 ACTION=passthru ARGS=systemctl is-active --quiet elasticsearch@identity-sau-main-dev-node-01.service
✅ Service: ACTIVE
⏳ Waiting for TCP search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200 to be accessible (timeout 5s)...
✅ Port 9200 is accessible on search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com.
✅ Port: LISTENING on search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
✅ HTTP: RESPONDING on search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200
Node name: identity-sau-main-dev-node-01
========================================
🏥 Cluster Health Check
========================================
Cluster Name: fastorder-identity-sau-main-dev
Nodes Count: "number_of_nodes"
Status: green
[ OK ] ✅ Cluster status: GREEN (healthy)
Full cluster health:
{
"cluster_name" : "fastorder-identity-sau-main-dev",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 0,
"active_shards" : 0,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"unassigned_primary_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
========================================
📊 Final Verification Results
========================================
[ OK ] ✅ Comprehensive verification PASSED!
[ OK ] Environment 'identity-sau-main-dev' with 1 nodes is fully operational
📋 QUICK DIAGNOSTIC COMMANDS:
----------------------------------------
# Test cluster endpoints:
curl http://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
# Check cluster health:
curl http://search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200/_cluster/health?pretty
# Check nodes info:
curl http://search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com:9200/_cat/nodes?v
# Check all Elasticsearch ports:
sudo ss -tlnp | grep java
# Check systemd service status:
sudo /usr/local/bin/fastorder-provisioning-wrapper.sh passthru systemctl status elasticsearch@identity-sau-main-dev-node-01.service
# View recent logs:
sudo journalctl -u elasticsearch@identity-sau-main-dev-node-01.service -f
[INFO] Environment: identity-sau-main-dev
[INFO] Nodes: 1
[INFO] Port: 9200 (default Elasticsearch port)
[INFO] Coordinator endpoint: http://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
=== Elasticsearch HTTP Setup completed successfully! ===
Environment: (1 nodes)
Port: 9200 (default Elasticsearch port)
✅ Coordinator endpoint: http://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
Quick test commands:
curl http://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
curl http://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/_cluster/health?pretty
[0;32m[1m✓ Step 1 completed successfully![0m
[0;36m[1m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;32m[1mStep 2: Executing Make Https[0m
[0;35mFolder: 02-make-https[0m
[0;36m[1m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
=== Elasticsearch HTTPS Setup ===
Configure HTTPS/SSL for Elasticsearch cluster
[INFO] Using web-provided environment: identity-sau-main-dev
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment:
Nodes: 1
Node: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
Port: 9200 (default port)
Executing: steps/01-generate-ssl-certificates.sh
==================================================================
STEP 1: Generate SSL certificates for Elasticsearch transport
==================================================================
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Domain: identity-sau-main-dev.fastorder.com
Environment: identity-sau-main-dev
Nodes: 1
Per-node VM IPs and domains:
Node 1: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
Port: 9200 (default port for all nodes)
=== Generating SSL certificates for ES transport ===
[INFO] Generating certificates for environment: identity-sau-main-dev (1 nodes)
[INFO] Configuring certificates for 1 node(s)
[INFO] Certificate storage: /etc/fastorder/elasticsearch/certs/identity-sau-main-dev
[2026-01-02 06:44:22 UTC] USER=www-data EUID=0 PID=1567028 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/temp-1566996
[2026-01-02 06:44:22 UTC] USER=www-data EUID=0 PID=1567037 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/temp-1566996
[2026-01-02 06:44:22 UTC] USER=www-data EUID=0 PID=1567046 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/temp-1566996
[2026-01-02 06:44:22 UTC] USER=www-data EUID=0 PID=1567065 ACTION=fsop ARGS=chmod 644 /etc/elasticsearch/temp-1566996/instances.yml
[INFO] Creating certificate instances configuration...
Adding node: identity-sau-main-dev-node-01 (search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com, 10.100.1.103)
[INFO] Certificate instances configuration:
instances:
- name: identity-sau-main-dev-node-01
dns: [ "identity-sau-main-dev-node-01", "localhost", "search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com" ]
ip: [ "10.100.1.103", "127.0.0.1" ]
[INFO] Creating Certificate Authority for identity-sau-main-dev...
[2026-01-02 06:44:22 UTC] USER=www-data EUID=0 PID=1567090 ACTION=fsop ARGS=mkdir -p /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs
[2026-01-02 06:44:23 UTC] USER=www-data EUID=0 PID=1567099 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/fastorder/elasticsearch/certs/identity-sau-main-dev
[2026-01-02 06:44:23 UTC] USER=www-data EUID=0 PID=1567108 ACTION=fsop ARGS=chmod -R 755 /etc/fastorder/elasticsearch/certs/identity-sau-main-dev
[2026-01-02 06:44:23 UTC] USER=www-data EUID=0 PID=1567117 ACTION=fsop ARGS=rm -f /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs/identity-sau-main-dev-ca.zip
yes: standard output: Broken pipe
[ OK ] ✓ CA certificate created
[INFO] Creating node certificates for identity-sau-main-dev...
yes: standard output: Broken pipe
[ OK ] ✓ Node certificates created
[INFO] Distributing certificates...
Configuring certificates for node 1 (identity-sau-main-dev-node-01)...
[2026-01-02 06:44:32 UTC] USER=www-data EUID=0 PID=1567309 ACTION=fsop ARGS=cp /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs/ca/ca.crt /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/ca.crt
[2026-01-02 06:44:33 UTC] USER=www-data EUID=0 PID=1567329 ACTION=fsop ARGS=cp /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs/nodes/identity-sau-main-dev-node-01.crt /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/
[2026-01-02 06:44:33 UTC] USER=www-data EUID=0 PID=1567338 ACTION=fsop ARGS=cp /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs/nodes/identity-sau-main-dev-node-01.key /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/
[2026-01-02 06:44:33 UTC] USER=www-data EUID=0 PID=1567347 ACTION=fsop ARGS=chmod 644 /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/identity-sau-main-dev-node-01.crt
[2026-01-02 06:44:33 UTC] USER=www-data EUID=0 PID=1567356 ACTION=fsop ARGS=chmod 600 /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/identity-sau-main-dev-node-01.key
[ OK ] ✓ Certificates copied for identity-sau-main-dev-node-01
[2026-01-02 06:44:33 UTC] USER=www-data EUID=0 PID=1567365 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01
[2026-01-02 06:44:33 UTC] USER=www-data EUID=0 PID=1567374 ACTION=fsop ARGS=find /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs -type f -name *.key -exec chmod 600 {} ;
[2026-01-02 06:44:33 UTC] USER=www-data EUID=0 PID=1567385 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs
[2026-01-02 06:44:33 UTC] USER=www-data EUID=0 PID=1567395 ACTION=fsop ARGS=rm -rf /etc/elasticsearch/temp-1566996
[ OK ] ✓ Certificates ready for environment: identity-sau-main-dev
[ OK ] ✓ SSL certificate generation completed successfully!
[INFO] Environment: identity-sau-main-dev
[INFO] Nodes configured: 1
[INFO] Per-node VM IPs and domains (each with default port 9200):
Node 1: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
[INFO] Certificate directory: /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs
[INFO] === Certificate Summary ===
CA Certificate: /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs/ca/ca.crt
Node Certificates:
- identity-sau-main-dev-node-01: /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/
[INFO] === Verification Commands ===
# Verify CA certificate:
openssl x509 -in /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/certs/ca/ca.crt -text -noout
# Verify node certificates:
openssl x509 -in /etc/fastorder/elasticsearch/certs/identity-sau-main-dev/node-01/identity-sau-main-dev-node-01.crt -text -noout
[INFO] Next: Configure transport SSL in Elasticsearch configuration files
Executing: steps/02-enable-security-transport.sh
==================================================================
STEP 2: Enable security with transport SSL
==================================================================
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Nodes: 1
Node: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
[INFO] === Single-Node Setup ===
[INFO] Enabling security (xpack.security.enabled: true)
[2026-01-02 06:44:34 UTC] USER=www-data EUID=0 PID=1567432 ACTION=fsop ARGS=sed -i /^xpack.security.enabled:/d /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.yml
[INFO] Disabling transport SSL (not needed for single-node)
[2026-01-02 06:44:34 UTC] USER=www-data EUID=0 PID=1567451 ACTION=fsop ARGS=sed -i /^xpack.security.transport.ssl.enabled:/d /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.yml
[ OK ] ==================================================================
[ OK ] Security and Transport SSL Configuration Complete
[ OK ] ==================================================================
[INFO] Environment: identity-sau-main-dev
[INFO] Nodes: 1
[INFO] Security enabled: true
[INFO] Transport SSL enabled: false (not required for single-node)
[INFO] === Next Step ===
Restart services to apply security configuration (step 04)
Executing: steps/03-http-ssl.sh
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
==================================================================
Direct HTTPS Configuration (native TLS, PEM)
==================================================================
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Nodes: 1
Node: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
Port: 9200 (default port)
Domain: identity-sau-main-dev.fastorder.com
[INFO] === Single-Node Direct HTTPS Setup ===
🔐 Generated PKCS12 password: iBKUR0Rv... (32 chars)
[INFO] Using first node configuration: /etc/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:44:35 UTC] USER=www-data EUID=0 PID=1567528 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:44:35 UTC] USER=www-data EUID=0 PID=1567538 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01
[INFO] Checking prerequisites...
[ OK ] ✓ Prerequisites verified
[INFO] Setting up temporary directories...
[2026-01-02 06:44:35 UTC] USER=www-data EUID=0 PID=1567552 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:44:35 UTC] USER=www-data EUID=0 PID=1567580 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:44:36 UTC] USER=www-data EUID=0 PID=1567612 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev
[2026-01-02 06:44:36 UTC] USER=www-data EUID=0 PID=1567648 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:44:36 UTC] USER=www-data EUID=0 PID=1567676 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:44:36 UTC] USER=www-data EUID=0 PID=1567685 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out
[ OK ] ✓ Directories created
[INFO] Building certificate instances configuration...
[INFO] Certificate instances configuration:
instances:
- name: "identity-sau-main-dev-http"
dns: [ "localhost", "web-03", "search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com", "identity-sau-main-dev-node-01.fastorder.com", "search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com", "search-identity-sau-main-dev.fastorder.com", "identity-sau-main-dev-node-01.local" ]
ip: [ "10.100.1.103", "127.0.0.1", "::1" ]
[ OK ] ✓ Instances configuration created
[INFO] Generating HTTP Certificate Authority...
[2026-01-02 06:44:36 UTC] USER=www-data EUID=0 PID=1567704 ACTION=fsop ARGS=rm -f /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/http-ca.zip /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http-certs.zip
[2026-01-02 06:44:36 UTC] USER=www-data EUID=0 PID=1567713 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:44:36 UTC] USER=www-data EUID=0 PID=1567722 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:44:40 UTC] USER=www-data EUID=0 PID=1567806 ACTION=fsop ARGS=chmod 644 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/http-ca.zip
Archive: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/http-ca.zip
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/ca/ca.crt
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/ca/ca.key
[ OK ] ✓ HTTP CA generated successfully
[INFO] Generating per-node HTTP certificates...
[2026-01-02 06:44:40 UTC] USER=www-data EUID=0 PID=1567819 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out
[2026-01-02 06:44:40 UTC] USER=www-data EUID=0 PID=1567828 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567905 ACTION=fsop ARGS=chmod 644 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http-certs.zip
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567914 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http
Archive: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http-certs.zip
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http/identity-sau-main-dev-http/identity-sau-main-dev-http.crt
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http/identity-sau-main-dev-http/identity-sau-main-dev-http.key
[ OK ] ✓ HTTP certificates generated successfully
[INFO] Installing certificates and configuring services...
[INFO] Configuring main service for single-node HTTPS...
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567926 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01/certs
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567935 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/certs
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567944 ACTION=fsop ARGS=cp /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http/identity-sau-main-dev-http/identity-sau-main-dev-http.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http.crt
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567953 ACTION=fsop ARGS=cp /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http/identity-sau-main-dev-http/identity-sau-main-dev-http.key /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http.key
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567962 ACTION=fsop ARGS=cp /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/ca/ca.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567971 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/fastorder_ra_root.crt
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567980 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/certs
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567989 ACTION=fsop ARGS=chmod 600 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http.key
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1567998 ACTION=fsop ARGS=chmod 644 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/fastorder_ra_root.crt
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1568012 ACTION=fsop ARGS=sed -i -E -e /^\s*xpack\.security\.http\.ssl(\..*)?\s*:/d /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.yml
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1568022 ACTION=fsop ARGS=sed -i /^# --- BEGIN direct HTTPS (managed, PEM) ---$/,/^# --- END direct HTTPS (managed, PEM) ---$/d /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.yml
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1568031 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/fastorder_ra_root.crt
YAML: /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.yml
[ OK ] ✓ Main service configured with HTTPS
[2026-01-02 06:44:45 UTC] USER=www-data EUID=0 PID=1568068 ACTION=fsop ARGS=chown -R elasticsearch:sslusers /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients
Archive: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client.zip
creating: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key
[INFO] Creating P12 keystore for es-client...
[2026-01-02 06:44:50 UTC] USER=www-data EUID=0 PID=1568244 ACTION=fsop ARGS=mv /tmp/es-client-1567497.p12 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.p12
[ OK ] ✓ Created P12 keystore: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.p12
[2026-01-02 06:44:50 UTC] USER=www-data EUID=0 PID=1568257 ACTION=fsop ARGS=chown -R elasticsearch:sslusers /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients
[2026-01-02 06:44:50 UTC] USER=www-data EUID=0 PID=1568266 ACTION=fsop ARGS=chmod 640 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key
[2026-01-02 06:44:50 UTC] USER=www-data EUID=0 PID=1568276 ACTION=fsop ARGS=chmod 600 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.p12
[2026-01-02 06:44:50 UTC] USER=www-data EUID=0 PID=1568285 ACTION=fsop ARGS=chmod 644 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt
[INFO] Saving keystore passwords to secrets vault...
🔑 Configuring AWS credentials for secrets vault...
✅ Using permanent AWS credentials from /home/ab/.aws/credentials
[0;34m[INFO][0m 🔐 Vaulting search passwords to remote backend...
[0;32m✅ Passwords vaulted to remote backend[0m
[0;32m✓ Keystore passwords saved to secrets vault: search/identity-sau-main-dev/keystore-passwords[0m
[0;34m[INFO][0m === Installing CA Certificate for Users ===
[0;34m[INFO][0m HOME not set, skipping user CA installation
[0;32m✓ Direct HTTPS configuration completed for environment: identity-sau-main-dev[0m
[0;34m[INFO][0m All services now serve HTTPS using PEM certificates
[0;34m[INFO][0m Network binding: 10.100.1.103
[0;34m[INFO][0m HTTPS endpoint: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[0;34m[INFO][0m === Certificate Summary ===
CA Certificate: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/ca/ca.crt
Certificate Directory: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
Main service certificates: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/
[0;34m[INFO][0m === Next Steps ===
1. Restart Elasticsearch services to apply HTTPS configuration
2. Test HTTPS connectivity: curl https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
3. Verify certificates: openssl s_client -connect search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[0;34m[INFO][0m === SSL Certificate Information ===
• CA certificate installed for user - curl will work without -k flag
• Each certificate includes node-specific domain, VM IP, and localhost in Subject Alternative Names
• Certificates are valid for 3 years from generation date
[1;33m[WARNING][0m Important: You'll need to restart Elasticsearch services for HTTPS to take effect
Executing: steps/04-restart-systemd-services.sh
==================================================================
STEP 4 (STRICT): Restart systemd services and verify secure health
==================================================================
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
Environment: identity-sau-main-dev
Nodes: 1
Per-node endpoints (all use default port 9200):
Node 1: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
[INFO] Building service list for environment: identity-sau-main-dev (1 nodes)
- elasticsearch@identity-sau-main-dev-node-01.service (port 9200)
[INFO] Will restart 1 service(s) for environment: identity-sau-main-dev
[2026-01-02 06:44:57 UTC] USER=www-data EUID=0 PID=1568417 ACTION=passthru ARGS=systemctl daemon-reload
[INFO] === Ensuring VM IPs are configured correctly ===
[INFO] ✓ 10.100.1.103 already configured on eth0 for node-01
[INFO] === Ensuring transport SSL certificates for all nodes ===
[INFO] ✓ Transport certificates already exist for node-01
[INFO] === Restarting Services ===
↻ Restarting elasticsearch@identity-sau-main-dev-node-01.service ...
[2026-01-02 06:44:58 UTC] USER=www-data EUID=0 PID=1568482 ACTION=passthru ARGS=systemctl restart elasticsearch@identity-sau-main-dev-node-01.service
[2026-01-02 06:45:03 UTC] USER=www-data EUID=0 PID=1569358 ACTION=passthru ARGS=systemctl is-active --quiet elasticsearch@identity-sau-main-dev-node-01.service
[ OK ] elasticsearch@identity-sau-main-dev-node-01.service is active
[INFO] Waiting 10s for Elasticsearch to start listening on ports...
[INFO] === Waiting for STRICT Secure Cluster Health ===
[INFO] Waiting for port 9200 on 10.100.1.103 (timeout 120s)...
[INFO] Waiting for cluster to form and be ready for write operations...
✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓
[INFO] Cluster stable and ready for operations (20 consecutive healthy responses over 40s)
[INFO] Performing final cluster health check before password setup...
[INFO] Elastic password not found, running password setup...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[0;34m╔════════════════════════════════════════════════════════════╗[0m
[0;34m║ Elasticsearch Password Management via AWS Secrets MGR ║[0m
[0;34m╚════════════════════════════════════════════════════════════╝[0m
[0;34mEnvironment: identity-sau-main-dev[0m
[0;34mUser: elastic[0m
[0;34mIdentifier: node-01[0m
[0;34mAWS Secret: fastorder/search/identity/sau/main/dev/elasticsearch/node-01[0m
Using configuration path: /etc/elasticsearch/identity-sau-main-dev/node-01 (IDENTIFIER: node-01)
Node domain: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com
HTTP port: 9200 (default Elasticsearch port)
[INFO] xpack.security.enabled already true → no restart.
[INFO] No restart needed.
[2026-01-02 06:47:22 UTC] USER=www-data EUID=0 PID=1578256 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:47:22 UTC] USER=www-data EUID=0 PID=1578283 ACTION=fsop ARGS=chown elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/users /etc/elasticsearch/identity-sau-main-dev/node-01/users_roles
[2026-01-02 06:47:22 UTC] USER=www-data EUID=0 PID=1578292 ACTION=fsop ARGS=chmod 660 /etc/elasticsearch/identity-sau-main-dev/node-01/users /etc/elasticsearch/identity-sau-main-dev/node-01/users_roles
[0;32m✓ users/users_roles present and writable[0m
[2026-01-02 06:47:22 UTC] USER=www-data EUID=0 PID=1578301 ACTION=fsop ARGS=chown elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.keystore
[2026-01-02 06:47:22 UTC] USER=www-data EUID=0 PID=1578316 ACTION=fsop ARGS=chmod 660 /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.keystore
[0;32m✓ Keystore exists: /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.keystore[0m
[0;34mHTTPS is enabled in configuration[0m
[0;32m✓ Found HTTP CA certificate: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt[0m
[0;32m✓ Using client certificates for mTLS[0m
[0;34mWaiting for Elasticsearch to be reachable at https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200...[0m
[0;32m✓ Elasticsearch is reachable (HTTP 401)[0m
[0;34mES_PATH_CONF: /etc/elasticsearch/identity-sau-main-dev/node-01[0m
[0;34mHTTP URL: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200[0m
[0;34mRunning HTTP reset (online, --batch)…[0m
[0;34mNote: Using HTTPS - tools will read SSL config from elasticsearch.yml[0m
Command output:
Password for the [elastic] user successfully reset.
New value: T+kMy0e84aGeV204NzYK
Exit status: 0
[0;32m✓ HTTP reset succeeded for elastic[0m
[0;34mStoring credentials in AWS Secrets Manager: fastorder/search/identity/sau/main/dev/elasticsearch/node-01[0m
ℹ️ Setting Elasticsearch credentials in vault: fastorder/search/identity/sau/main/dev/elasticsearch/node-01
ℹ️ Setting secret in AWS Secrets Manager: fastorder/search/identity/sau/main/dev/elasticsearch/node-01
✅ Secret created: fastorder/search/identity/sau/main/dev/elasticsearch/node-01
✅ Elasticsearch credentials set in vault: fastorder/search/identity/sau/main/dev/elasticsearch/node-01
[0;32m✓ Password stored in AWS Secrets Manager: fastorder/search/identity/sau/main/dev/elasticsearch/node-01[0m
[0;32m✓ Cache cleared for: fastorder/search/identity/sau/main/dev/elasticsearch/node-01[0m
[0;32m✓ Done. Password stored in AWS Secrets Manager: fastorder/search/identity/sau/main/dev/elasticsearch/node-01[0m
Usage Examples:
# Retrieve password using AWS CLI
aws secretsmanager get-secret-value --secret-id fastorder/search/identity/sau/main/dev/elasticsearch/node-01 --region ${AWS_REGION:-me-central-1}
# Using fastctl
fastctl secrets get fastorder/search/identity/sau/main/dev/elasticsearch/node-01
# Test connection
curl -u elastic:$(fastctl secrets get fastorder/search/identity/sau/main/dev/elasticsearch/node-01 --field password) https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/_cluster/health
✓ Retrieved password from AWS Secrets Manager
[INFO] Testing cluster at: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[INFO] Using SSL CA certificate: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt
[INFO] Using client cert/key for mTLS
[INFO] Using client cert/key for mTLS: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt / /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key
[INFO] ⏳ waiting for secure cluster health (require 200) at https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200 (timeout 30s)...
[ OK ] Cluster health OK: green
[ OK ] Authentication successful with existing password
==================================================================
[ OK ] All services restarted successfully!
[ OK ] Cluster is healthy, HTTPS-secure, and responding with 200
[INFO] Environment: identity-sau-main-dev
[INFO] Services: 1
[INFO] Endpoint: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[INFO] === Manual verification (copy/paste) ===
curl -u 'elastic:T+kMy0e84aGeV204NzYK' \
--cacert '/etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt' \
--cert '/etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt' \
--key '/etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key' \
'https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/_cluster/health?pretty'
[INFO] === Quick checks ===
curl -u 'elastic:T+kMy0e84aGeV204NzYK' --cacert '/etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt' --cert '/etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt' --key '/etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key' https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/_cat/nodes?v
curl -u 'elastic:T+kMy0e84aGeV204NzYK' --cacert '/etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt' --cert '/etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt' --key '/etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key' https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/
Executing: steps/05-test-elastic.sh
==================================================================
STEP 5: Test Elasticsearch Cluster
==================================================================
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
Environment: identity-sau-main-dev
Nodes: 1
Node: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
Port: 9200 (default port)
[INFO] Using centralized test suite: /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/02-make-https/lib/elasticsearch-test-suite.sh
[INFO] Executing centralized test suite with args: -v -t all --env identity-sau-main-dev -u elastic
[0;34m[INFO][0m Using CURRENT_ENV_ID from environment: identity-sau-main-dev
[0;34m[INFO][0m Loaded from topology.json: identity-sau-main-dev
[0;32m[2026-01-02 06:47:44][0m Loaded environment: identity-sau-main-dev
[0;32m[2026-01-02 06:47:44][0m Service: identity, Zone: sau, Branch: main, Env: dev
[0;32m[2026-01-02 06:47:44][0m VM IP: 142.93.238.16, Interface: eth0:16
[0;32m[2026-01-02 06:47:44][0m Elasticsearch Nodes: 1, PostgreSQL Workers: 1
[0;32m[2026-01-02 06:47:45][0m PostgreSQL HA Nodes: 1, Citus Enabled: yes
[0;32m✓[0m Environment initialized successfully (mode: general)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[0;34mℹ[0m Using CURRENT_ENV_ID from environment: identity-sau-main-dev
[0;34mℹ[0m Loaded from topology.json: identity-sau-main-dev
[0;32m[2026-01-02 06:47:45][0m Loaded environment: identity-sau-main-dev
[0;32m[2026-01-02 06:47:45][0m Service: identity, Zone: sau, Branch: main, Env: dev
[0;32m[2026-01-02 06:47:45][0m VM IP: 142.93.238.16, Interface: eth0:16
[0;32m[2026-01-02 06:47:45][0m Elasticsearch Nodes: 1, PostgreSQL Workers: 1
[0;32m[2026-01-02 06:47:45][0m PostgreSQL HA Nodes: 1, Citus Enabled: yes
[0;32m✓[0m Environment initialized successfully (mode: general)
[0;34mℹ[0m Project root: /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch
[0;34mℹ[0m Environment: identity-sau-main-dev
[0;34mℹ[0m Nodes count: 1
[0;34mℹ[0m Endpoint: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[0;34mℹ[0m Using CA: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt
[0;34mℹ[0m Using mTLS: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt / /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key
[0;34m╔════════════════════════════════════════════╗[0m
[0;34m║ Elasticsearch Centralized Test Suite ║[0m
[0;34m╚════════════════════════════════════════════╝[0m
[0;34m=== Authentication Test ===[0m
[0;32m✓[0m Loaded credentials for user elastic from AWS Secrets Manager
[0;34mCurl (local):[0m curl --cacert /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt --cert /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt --key /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key -u 'elastic:********' 'https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/_cluster/health?pretty'
[0;32m✓[0m Local authentication successful (HTTP 200).
{
"cluster_name" : "fastorder-identity-sau-main-dev",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 3,
"active_shards" : 3,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"unassigned_primary_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
Executing: steps/06-final-testing.sh
==================================================================
STEP 6: Final Testing and Verification
==================================================================
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
Environment: identity-sau-main-dev
Nodes: 1
Node: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
Port: 9200 (default port)
[INFO] Using centralized test suite: /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/02-make-https/lib/elasticsearch-test-suite.sh
[0;34m[INFO][0m Using CURRENT_ENV_ID from environment: identity-sau-main-dev
[0;34m[INFO][0m Loaded from topology.json: identity-sau-main-dev
[0;32m[2026-01-02 06:47:49][0m Loaded environment: identity-sau-main-dev
[0;32m[2026-01-02 06:47:49][0m Service: identity, Zone: sau, Branch: main, Env: dev
[0;32m[2026-01-02 06:47:49][0m VM IP: 142.93.238.16, Interface: eth0:16
[0;32m[2026-01-02 06:47:49][0m Elasticsearch Nodes: 1, PostgreSQL Workers: 1
[0;32m[2026-01-02 06:47:49][0m PostgreSQL HA Nodes: 1, Citus Enabled: yes
[0;32m✓[0m Environment initialized successfully (mode: general)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[0;34mℹ[0m Using CURRENT_ENV_ID from environment: identity-sau-main-dev
[0;34mℹ[0m Loaded from topology.json: identity-sau-main-dev
[0;32m[2026-01-02 06:47:50][0m Loaded environment: identity-sau-main-dev
[0;32m[2026-01-02 06:47:50][0m Service: identity, Zone: sau, Branch: main, Env: dev
[0;32m[2026-01-02 06:47:50][0m VM IP: 142.93.238.16, Interface: eth0:16
[0;32m[2026-01-02 06:47:50][0m Elasticsearch Nodes: 1, PostgreSQL Workers: 1
[0;32m[2026-01-02 06:47:50][0m PostgreSQL HA Nodes: 1, Citus Enabled: yes
[0;32m✓[0m Environment initialized successfully (mode: general)
[0;34mℹ[0m Project root: /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch
[0;34mℹ[0m Environment: identity-sau-main-dev
[0;34mℹ[0m Nodes count: 1
[0;34mℹ[0m Endpoint: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[0;34mℹ[0m Using CA: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt
[0;34mℹ[0m Using mTLS: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt / /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key
[0;34m╔════════════════════════════════════════════╗[0m
[0;34m║ Elasticsearch Centralized Test Suite ║[0m
[0;34m╚════════════════════════════════════════════╝[0m
[0;34m=== Authentication Test ===[0m
[0;32m✓[0m Loaded credentials for user elastic from AWS Secrets Manager
[0;34mCurl (local):[0m curl --cacert /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt --cert /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt --key /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key -u 'elastic:********' 'https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/_cluster/health?pretty'
[0;32m✓[0m Local authentication successful (HTTP 200).
{
"cluster_name" : "fastorder-identity-sau-main-dev",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 3,
"active_shards" : 3,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"unassigned_primary_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
Executing: steps/07-set-passwords.sh
==================================================================
STEP 7: Setting cluster passwords (bootstrap via alias)
==================================================================
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Using HTTPS with CA: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt (host: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com)
[INFO] Using centralized password setter: /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/02-make-https/steps/../lib/elasticsearch-set-password.sh
[ OK ] Elastic password already valid (HTTP 200) via search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com; nothing to do.
Executing: steps/08-create-app-user.sh
==================================================================
STEP 8: Create Application User and Roles (cluster-scoped)
==================================================================
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
Environment: identity-sau-main-dev
Nodes: 1
[INFO] Using HTTPS with CA: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt (host: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com)
[ OK ] Retrieved elastic password from Vault (cluster scope).
[INFO] Configuration:
[INFO] App User : app_user
[INFO] Read-only Role : app_ro
[INFO] Read-write Role : app_rw
[INFO] Index Patterns : app-*,cdc-*,identity_sau_*,*_account_router
[INFO] Endpoint : https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[INFO] Creating read-only role: app_ro
[ OK ] ✓ Role app_ro ensured
[INFO] Creating read-write role: app_rw
[ OK ] ✓ Role app_rw ensured
[INFO] Creating/Updating application user: app_user
[ OK ] ✓ User app_user ensured
ℹ️ Setting Elasticsearch credentials in vault: fastorder/search/identity/sau/main/dev/elasticsearch/node-01/app_user
ℹ️ Setting secret in AWS Secrets Manager: fastorder/search/identity/sau/main/dev/elasticsearch/node-01/app_user
✅ Secret created: fastorder/search/identity/sau/main/dev/elasticsearch/node-01/app_user
✅ Elasticsearch credentials set in vault: fastorder/search/identity/sau/main/dev/elasticsearch/node-01/app_user
[ OK ] ✓ Stored app_user password under 'node-01/app_user'
ℹ️ Setting Elasticsearch credentials in vault: fastorder/search/identity/sau/main/dev/elasticsearch/cluster/app_user
ℹ️ Setting secret in AWS Secrets Manager: fastorder/search/identity/sau/main/dev/elasticsearch/cluster/app_user
✅ Secret created: fastorder/search/identity/sau/main/dev/elasticsearch/cluster/app_user
✅ Elasticsearch credentials set in vault: fastorder/search/identity/sau/main/dev/elasticsearch/cluster/app_user
[ OK ] ✓ Stored app_user password under 'cluster/app_user'
[INFO] Testing authentication for app_user...
[ OK ] ✓ Authentication test passed for app_user
[ OK ] ✓ Application user and roles created successfully!
[INFO] User : app_user
[INFO] Roles : app_ro, app_rw
[INFO] Patterns: app-*,cdc-*,identity_sau_*,*_account_router
[INFO] Endpoint: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
Executing: steps/09-config.sh
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Nodes: 1
Node: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
Port: 9200
[0;32m✓[0m Auto mode: Cloud IMDS detected → MODE=role
[0;34m[INFO][0m Mode: role
[0;34m[INFO][0m AWS Region: me-central-1
[0;34m[INFO][0m MODE=role → will purge any static S3 keys from each node keystore
[2026-01-02 06:49:37 UTC] USER=www-data EUID=0 PID=1581165 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01
[0;34m[INFO][0m • node-01 keystore cleared (role-based auth)
[2026-01-02 06:49:48 UTC] USER=www-data EUID=0 PID=1581454 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:49:50 UTC] USER=www-data EUID=0 PID=1581512 ACTION=passthru ARGS=systemctl restart elasticsearch@identity-sau-main-dev-node-01.service
[0;32m✓[0m ✓ restarted elasticsearch@identity-sau-main-dev-node-01.service
⏳ Waiting for HTTPS readiness on https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[INFO] Waiting HTTP readiness at https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/ (200/401/302)…
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
Waiting ...
[OK] Ready: 401
⏳ Waiting for cluster health (green|yellow)
[INFO] Waiting health (green|yellow) at https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/_cluster/health…
[OK] 401 pre-auth received; security enabled.
[0;32m✓[0m ✓ identity-sau-main-dev is responding via search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com
[0;32m✓[0m ✓ AWS S3 configuration completed for environment: identity-sau-main-dev (1 nodes)
[0;34m[INFO][0m Mode: role
[0;34m[INFO][0m Region: me-central-1
Executing: steps/0ld-03-http-ssl.sh
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
==================================================================
STEP 5: HTTP SSL Configuration (Optional)
==================================================================
Environment: identity-sau-main-dev
Nodes: 1
Node: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
Port: 9200 (default port)
[ OK ] 🚀 Auto mode/Default installation: Selecting Direct HTTPS configuration (option 1)
[ OK ] Configuring Direct HTTPS (Elasticsearch native SSL)...
──────────────────────────────────────────────────────────
[INFO] Environment: identity-sau-main-dev (1 nodes)
[INFO] Node: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
[INFO] Port: 9200 (default port)
==================================================================
Direct HTTPS Configuration (native TLS, PEM)
==================================================================
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Nodes: 1
Node: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com (10.100.1.103)
Port: 9200 (default port)
Domain: identity-sau-main-dev.fastorder.com
[INFO] === Single-Node Direct HTTPS Setup ===
🔐 Generated PKCS12 password: 8siDJx7z... (32 chars)
[INFO] Using first node configuration: /etc/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:50:46 UTC] USER=www-data EUID=0 PID=1582658 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:50:46 UTC] USER=www-data EUID=0 PID=1582671 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01
[INFO] Checking prerequisites...
[ OK ] ✓ Prerequisites verified
[INFO] Setting up temporary directories...
[2026-01-02 06:50:47 UTC] USER=www-data EUID=0 PID=1582716 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:50:47 UTC] USER=www-data EUID=0 PID=1582769 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:50:47 UTC] USER=www-data EUID=0 PID=1582802 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev
[2026-01-02 06:50:47 UTC] USER=www-data EUID=0 PID=1582812 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01
[2026-01-02 06:50:47 UTC] USER=www-data EUID=0 PID=1582821 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:50:47 UTC] USER=www-data EUID=0 PID=1582830 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out
[ OK ] ✓ Directories created
[INFO] Building certificate instances configuration...
[INFO] Certificate instances configuration:
instances:
- name: "identity-sau-main-dev-http"
dns: [ "localhost", "web-03", "search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com", "identity-sau-main-dev-node-01.fastorder.com", "search-identity-sau-main-dev-elasticsearch-coordinator.fastorder.com", "search-identity-sau-main-dev.fastorder.com", "identity-sau-main-dev-node-01.local" ]
ip: [ "10.100.1.103", "127.0.0.1", "::1" ]
[ OK ] ✓ Instances configuration created
[INFO] Generating HTTP Certificate Authority...
[2026-01-02 06:50:47 UTC] USER=www-data EUID=0 PID=1582854 ACTION=fsop ARGS=rm -f /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/http-ca.zip /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http-certs.zip
[2026-01-02 06:50:47 UTC] USER=www-data EUID=0 PID=1582863 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:50:47 UTC] USER=www-data EUID=0 PID=1582872 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
[2026-01-02 06:50:53 UTC] USER=www-data EUID=0 PID=1582956 ACTION=fsop ARGS=chmod 644 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/http-ca.zip
Archive: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/http-ca.zip
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/ca/ca.crt
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/ca/ca.key
[ OK ] ✓ HTTP CA generated successfully
[INFO] Generating per-node HTTP certificates...
[2026-01-02 06:50:53 UTC] USER=www-data EUID=0 PID=1582968 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out
[2026-01-02 06:50:53 UTC] USER=www-data EUID=0 PID=1582977 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out
[2026-01-02 06:50:56 UTC] USER=www-data EUID=0 PID=1583041 ACTION=fsop ARGS=chmod 644 /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http-certs.zip
[2026-01-02 06:50:56 UTC] USER=www-data EUID=0 PID=1583050 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http
Archive: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http-certs.zip
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http/identity-sau-main-dev-http/identity-sau-main-dev-http.crt
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http/identity-sau-main-dev-http/identity-sau-main-dev-http.key
[ OK ] ✓ HTTP certificates generated successfully
[INFO] Installing certificates and configuring services...
[INFO] Configuring main service for single-node HTTPS...
[2026-01-02 06:50:56 UTC] USER=www-data EUID=0 PID=1583063 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01/certs
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583072 ACTION=fsop ARGS=chmod 755 /etc/elasticsearch/identity-sau-main-dev/node-01/certs
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583083 ACTION=fsop ARGS=cp /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http/identity-sau-main-dev-http/identity-sau-main-dev-http.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http.crt
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583092 ACTION=fsop ARGS=cp /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/out/http/identity-sau-main-dev-http/identity-sau-main-dev-http.key /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http.key
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583101 ACTION=fsop ARGS=cp /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/ca/ca.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583110 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/fastorder_ra_root.crt
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583119 ACTION=fsop ARGS=chown -R elasticsearch:elasticsearch /etc/elasticsearch/identity-sau-main-dev/node-01/certs
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583128 ACTION=fsop ARGS=chmod 600 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http.key
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583137 ACTION=fsop ARGS=chmod 644 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/fastorder_ra_root.crt
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583146 ACTION=fsop ARGS=sed -i -E -e /^\s*xpack\.security\.http\.ssl(\..*)?\s*:/d /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.yml
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583155 ACTION=fsop ARGS=sed -i /^# --- BEGIN direct HTTPS (managed, PEM) ---$/,/^# --- END direct HTTPS (managed, PEM) ---$/d /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.yml
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583164 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/fastorder_ra_root.crt
YAML: /etc/elasticsearch/identity-sau-main-dev/node-01/elasticsearch.yml
[ OK ] ✓ Main service configured with HTTPS
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583183 ACTION=fsop ARGS=rm -rf /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583192 ACTION=fsop ARGS=mkdir -p /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients
[2026-01-02 06:50:57 UTC] USER=www-data EUID=0 PID=1583202 ACTION=fsop ARGS=chown -R elasticsearch:sslusers /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients
Archive: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client.zip
creating: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt
inflating: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key
[INFO] Creating P12 keystore for es-client...
[2026-01-02 06:51:02 UTC] USER=www-data EUID=0 PID=1583299 ACTION=fsop ARGS=mv /tmp/es-client-1582623.p12 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.p12
[ OK ] ✓ Created P12 keystore: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.p12
[2026-01-02 06:51:02 UTC] USER=www-data EUID=0 PID=1583308 ACTION=fsop ARGS=chown -R elasticsearch:sslusers /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients
[2026-01-02 06:51:02 UTC] USER=www-data EUID=0 PID=1583317 ACTION=fsop ARGS=chmod 640 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key
[2026-01-02 06:51:02 UTC] USER=www-data EUID=0 PID=1583326 ACTION=fsop ARGS=chmod 600 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.p12
[2026-01-02 06:51:02 UTC] USER=www-data EUID=0 PID=1583335 ACTION=fsop ARGS=chmod 644 /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt
[INFO] Saving keystore passwords to secrets vault...
🔑 Configuring AWS credentials for secrets vault...
✅ Using permanent AWS credentials from /home/ab/.aws/credentials
[0;34m[INFO][0m 🔐 Vaulting search passwords to remote backend...
[0;32m✅ Passwords vaulted to remote backend[0m
[0;32m✓ Keystore passwords saved to secrets vault: search/identity-sau-main-dev/keystore-passwords[0m
[0;34m[INFO][0m === Installing CA Certificate for Users ===
[0;34m[INFO][0m HOME not set, skipping user CA installation
[0;32m✓ Direct HTTPS configuration completed for environment: identity-sau-main-dev[0m
[0;34m[INFO][0m All services now serve HTTPS using PEM certificates
[0;34m[INFO][0m Network binding: 10.100.1.103
[0;34m[INFO][0m HTTPS endpoint: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[0;34m[INFO][0m === Certificate Summary ===
CA Certificate: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs/ca/ca.crt
Certificate Directory: /etc/elasticsearch/identity-sau-main-dev/node-01/http-certs
Main service certificates: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/
[0;34m[INFO][0m === Next Steps ===
1. Restart Elasticsearch services to apply HTTPS configuration
2. Test HTTPS connectivity: curl https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
3. Verify certificates: openssl s_client -connect search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[0;34m[INFO][0m === SSL Certificate Information ===
• CA certificate installed for user - curl will work without -k flag
• Each certificate includes node-specific domain, VM IP, and localhost in Subject Alternative Names
• Certificates are valid for 3 years from generation date
[1;33m[WARNING][0m Important: You'll need to restart Elasticsearch services for HTTPS to take effect
[ OK ] ✓ Direct HTTPS configuration completed successfully
[ OK ] ==================================================================
[ OK ] HTTP SSL Configuration Complete
[ OK ] ==================================================================
[INFO] Environment: identity-sau-main-dev
[INFO] Nodes: 1
[INFO] Configuration applied to port: 9200 (default port for all nodes)
[INFO] === Next Steps ===
1. Verify Elasticsearch is running: systemctl status elasticsearch@identity-sau-main-dev-node-01.service
2. Test cluster health: curl https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200/_cluster/health
3. Check SSL certificate: openssl s_client -connect search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
=== HTTPS Setup completed successfully! ===
Environment: (1 nodes)
Domain: .fastorder.com
HTTPS endpoint: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
Node IP: 10.100.1.103
[0;32m[1m✓ Step 2 completed successfully![0m
[0;36m[1m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;32m[1mStep 3: Executing Create Index Llm[0m
[0;35mFolder: 03-create-index-llm[0m
[0;36m[1m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
==================================================================
Elasticsearch LLM/Semantic Search Setup
==================================================================
[INFO] Using web-provided environment: identity-sau-main-dev
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
Environment: identity-sau-main-dev
Service : identity
🔍 Checking Elasticsearch availability…
✅ Elasticsearch is accessible at https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
=== Phase 1: Common steps under /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps ===
(no numbered steps in: /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps)
=== Phase 2: Service-scoped steps for 'identity' under /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps/identity ===
📚 Detected features: login
── Feature: login
▶️ Running /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps/identity/login/01-create-model-and-pipeline.sh
==================================================================
STEP 1: Create Model and Ingest Pipeline
==================================================================
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] ES URL: https://search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com:9200
[INFO] Endpoint ID (ES): identity-text-embedding-001
[INFO] Provider model: text-embedding-3-large
[INFO] Pipeline ID: identity-embed-pipeline-001
[INFO] Checking authentication identity…
{
"username":"elastic","roles":["superuser"],"full_name":null,"email":null,"metadata":{"_reserved":true},"enabled":true,"authentication_realm":{"name":"reserved","type":"reserved"},"lookup_realm":{"name":"reserved","type":"reserved"},"authentication_type":"realm"
}
[INFO] Checking Elasticsearch license…
[INFO] License type: unknown
[WARN] Inference API requires Enterprise/Platinum license (found: unknown)
[WARN] Skipping inference endpoint and pipeline creation
[1;32m[OK][0m Setup completed (inference features skipped due to license)
✅ 01-create-model-and-pipeline.sh completed
▶️ Running /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps/identity/login/02-create-index.sh
==================================================================
STEP 2: Create Semantic Search Index (initial bootstrap)
==================================================================
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] [create] Ensuring clean slate for: identity_sau_main_dev_account_router-000001
[INFO] Index identity_sau_main_dev_account_router-000001 does not exist (status 404), proceeding.
[INFO] [create] Creating index identity_sau_main_dev_account_router-000001 with write alias identity_sau_main_dev_account_router
[1;32m[OK][0m Index + alias ready.
Index (concrete): identity_sau_main_dev_account_router-000001
Alias (stable) : identity_sau_main_dev_account_router (is_write_index=true)
Default pipeline: identity-embed-pipeline-001
Vector dims : 3072 (KNN cosine)
✅ 02-create-index.sh completed
▶️ Running /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps/identity/login/03-llm.sh
==================================================================
STEP 2: Create Semantic Search Index (ILM bootstrap)
==================================================================
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] [cluster] Ensure disk watermarks permit allocation
[1;32m[OK][0m Cluster watermarks set/confirmed.
[INFO] [create] Create/Update ILM policy: identity-account-router-ilm
[1;32m[OK][0m ILM policy ready.
[INFO] [create] Create/Update index template: identity_sau_main_dev_account_router_template
[1;32m[OK][0m Index template ready.
[INFO] [check] Concrete index: identity_sau_main_dev_account_router-000001
[1;32m[OK][0m Concrete index identity_sau_main_dev_account_router-000001 already exists (skip create).
[INFO] [verify] Wait for index to be at least YELLOW
[1;32m[OK][0m Cluster health OK for identity_sau_main_dev_account_router-000001.
[INFO] [verify] Alias points to a concrete write index
[1;32m[OK][0m Alias verification passed.
[INFO] [explain] ILM status
{
"indices" : {
"identity_sau_main_dev_account_router-000001" : {
"index" : "identity_sau_main_dev_account_router-000001",
"managed" : false
}
}
}
[1;32m[OK][0m ILM/alias bootstrap complete.
Index (concrete): identity_sau_main_dev_account_router-000001
Alias (stable) : identity_sau_main_dev_account_router (is_write_index=true)
ILM policy : identity-account-router-ilm
Default pipeline: identity-embed-pipeline-001
✅ 03-llm.sh completed
▶️ Running /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps/identity/login/04-index-sample-data.sh
==================================================================
STEP 3: Index Sample Data
==================================================================
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[WARN] Pipeline 'identity-embed-pipeline-001' not found (HTTP 404); proceeding without it.
[INFO] [bulk] Index seed documents → identity_sau_main_dev_account_router
[WARN] Bulk completed with item-level errors. Showing first 50 lines:
{"errors":true,"took":0,"ingest_took":0,"items":[{"index":{"_index":"identity_sau_main_dev_account_router","_id":"auto-generated","status":400,"error":{"type":"illegal_argument_exception","reason":"pipeline with id [identity-embed-pipeline-001] does not exist"}}},{"index":{"_index":"identity_sau_main_dev_account_router","_id":"auto-generated","status":400,"error":{"type":"illegal_argument_exception","reason":"pipeline with id [identity-embed-pipeline-001] does not exist"}}},{"index":{"_index":"identity_sau_main_dev_account_router","_id":"auto-generated","status":400,"error":{"type":"illegal_argument_exception","reason":"pipeline with id [identity-embed-pipeline-001] does not exist"}}}]}[summary] items=3 errors=3
[INFO] [verify] Search a sample term: 'password'
{
"took" : 136,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 0,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
}
}
[1;32m[OK][0m Sample data indexing step completed.
✅ 04-index-sample-data.sh completed
▶️ Running /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps/identity/login/05-create-cdc-index.sh
==================================================================
STEP 5: Create CDC Account Router Index (for dashboard visibility)
==================================================================
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Creating CDC index: identity_sau_main_dev_account_router
[1;32m[OK][0m Index identity_sau_main_dev_account_router already exists
✅ 05-create-cdc-index.sh completed
=== Phase 3: Optional search smoke tests ===
(semantic search test script not found: /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps/search-semantic.sh)
(hybrid search test script not found: /opt/fastorder/bash/scripts/env_app_setup/setup/03-search/engine/elasticsearch/03-create-index-llm/steps/hybrid-search.sh)
==================================================================
🎉 LLM/Semantic Search setup completed successfully!
==================================================================
Available commands:
• Test semantic search:
bash steps/search-semantic.sh en "password policy"
bash steps/search-semantic.sh ar "كلمة المرور"
• Test hybrid search:
bash steps/hybrid-search.sh en "user authentication"
bash steps/hybrid-search.sh ar "مصادقة المستخدم"
Alias : identity_sau_main_dev_account_router
Index : identity_sau_main_dev_account_router-000001
ILM : identity-account-router-ilm
Model : identity-text-embedding-001
Pipeline: identity-embed-pipeline-001
==================================================================
[0;32m[1m✓ Step 3 completed successfully![0m
[0;36m[1m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;32m[1mStep 4: Executing Monitoring Setup[0m
[0;35mFolder: 10-monitoring-setup[0m
[0;36m[1m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[INFO] Using web-provided environment: identity-sau-main-dev
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] 🔍 Elasticsearch Monitoring Integration for identity-sau-main-dev
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] 1️⃣ Checking observability cell readiness...
[INFO] Checking observability cell readiness: obs-identity-sau-main-dev
[1;32m[OK][0m Observability cell endpoints registered for identity-sau-main-dev
[1;32m[OK][0m ✓ Observability cell is ready
[INFO] 2️⃣ Discovering Elasticsearch configuration...
[1;32m[OK][0m ✓ Found Elasticsearch at 10.100.1.103:9200
[INFO] 3️⃣ Setting up elasticsearch_exporter integration...
[INFO] Using elasticsearch_exporter port: 9114
[INFO] SSL certificates configured for elasticsearch_exporter:
[INFO] CA cert: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/http_ca.crt
[INFO] Client cert: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.crt
[INFO] Client key: /etc/elasticsearch/identity-sau-main-dev/node-01/certs/clients/es-client/es-client.key
[INFO] Checking observability cell readiness: obs-identity-sau-main-dev
[1;32m[OK][0m Observability cell endpoints registered for identity-sau-main-dev
[INFO] Setting up elasticsearch_exporter for identity-sau-main-dev
[INFO] Elasticsearch exporter will bind to: 10.100.1.103:9114
[2026-01-02 06:51:32 UTC] USER=www-data EUID=0 PID=1584445 ACTION=passthru ARGS=mv /tmp/elasticsearch_exporter-identity-sau-main-dev.service /etc/systemd/system/elasticsearch_exporter-identity-sau-main-dev.service
[2026-01-02 06:51:32 UTC] USER=www-data EUID=0 PID=1584454 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:51:33 UTC] USER=www-data EUID=0 PID=1584510 ACTION=passthru ARGS=systemctl enable elasticsearch_exporter-identity-sau-main-dev.service
Created symlink /etc/systemd/system/multi-user.target.wants/elasticsearch_exporter-identity-sau-main-dev.service -> /etc/systemd/system/elasticsearch_exporter-identity-sau-main-dev.service.
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
IP Conflict Check
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
IP Address: 10.100.1.103
Port: 9114
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔍 Checking IP conflict for identity-sau-main-dev on 10.100.1.103:9114...
✅ IP 10.100.1.103:9114 is available - no conflicts detected
🔍 Checking for orphaned processes that might conflict...
✅ No orphaned processes detected
✅ All checks passed - safe to proceed with identity-sau-main-dev setup
[2026-01-02 06:51:34 UTC] USER=www-data EUID=0 PID=1584602 ACTION=passthru ARGS=systemctl restart elasticsearch_exporter-identity-sau-main-dev.service
[1;32m[OK][0m elasticsearch_exporter configured on 10.100.1.103:9114
[INFO] Register this endpoint in metrics-identity-sau-main-dev.fastorder.com scrape config
[1;32m[OK][0m ✓ elasticsearch_exporter integration complete
[INFO] 3.5️⃣ Configuring Prometheus to scrape Elasticsearch metrics...
[2026-01-02 06:51:37 UTC] USER=www-data EUID=0 PID=1584778 ACTION=passthru ARGS=grep -q job_name: 'elasticsearch' /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
[INFO] Adding Elasticsearch scrape target to Prometheus configuration...
[2026-01-02 06:51:37 UTC] USER=www-data EUID=0 PID=1584801 ACTION=fsop ARGS=cp /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml.backup-1767336697
[INFO] Created backup: /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml.backup-1767336697
[2026-01-02 06:51:37 UTC] USER=www-data EUID=0 PID=1584822 ACTION=passthru ARGS=sed -i /# Prometheus self-monitoring/r /tmp/prometheus_es_add.yml /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
[2026-01-02 06:51:37 UTC] USER=www-data EUID=0 PID=1584843 ACTION=passthru ARGS=grep -q job_name: 'elasticsearch' /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
[INFO] ✓ Elasticsearch job successfully inserted into config
[INFO] Validating Prometheus configuration with promtool...
Checking /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
SUCCESS: 1 rule files found
SUCCESS: /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml is valid prometheus config file syntax
Checking /etc/prometheus/obs-identity-sau-main-dev/rules/basic_alerts.yml
SUCCESS: 4 rules found
[1;32m[OK][0m ✓ Prometheus configuration validation PASSED
[1;32m[OK][0m ✓ Prometheus configuration updated successfully
[2026-01-02 06:51:37 UTC] USER=www-data EUID=0 PID=1584875 ACTION=passthru ARGS=systemctl is-active --quiet prometheus-obs-identity-sau-main-dev.service
[INFO] Reloading Prometheus configuration...
[2026-01-02 06:51:38 UTC] USER=www-data EUID=0 PID=1584896 ACTION=passthru ARGS=systemctl restart prometheus-obs-identity-sau-main-dev.service
[2026-01-02 06:51:41 UTC] USER=www-data EUID=0 PID=1584953 ACTION=passthru ARGS=systemctl is-active --quiet prometheus-obs-identity-sau-main-dev.service
[1;32m[OK][0m ✓ Prometheus reloaded successfully
[2026-01-02 06:51:41 UTC] USER=www-data EUID=0 PID=1584974 ACTION=fsop ARGS=rm -f /tmp/prometheus_es_add.yml
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] ✅ Elasticsearch Monitoring Setup Complete
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] Service: elasticsearch_exporter-identity-sau-main-dev.service
[INFO] Metrics: http://localhost:9114/metrics
[INFO] Prometheus: https://metrics-identity-sau-main-dev.fastorder.com:9090
[INFO] Grafana: https://dashboards-identity-sau-main-dev.fastorder.com
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] 4️⃣ Registering Elasticsearch nodes to monitoring database...
[INFO] Constructed FQDN: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com
[INFO] Registering: identity-sau-main-dev-node-01
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: Elasticsearch
[INFO] Identifier: identity-sau-main-dev-node-01
[INFO] Identifier Parent: cluster
[INFO] IP: 10.100.1.103
[INFO] Port: 9200
[INFO] FQDN: search-identity-sau-main-dev-elasticsearch-node-01.fastorder.com
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: 0c4bbf26-5554-4b0a-91b2-b9563a7e8fbf
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[1;32m[OK][0m ✓ Registered: identity-sau-main-dev-node-01
[1;32m[OK][0m ✓ Elasticsearch node registration completed successfully
[INFO] 5️⃣ Verifying monitoring integration...
[INFO] Checking elasticsearch_exporter service...
[1;32m[OK][0m ✓ elasticsearch_exporter-identity-sau-main-dev.service is ACTIVE
[INFO] Checking Prometheus service...
[1;32m[OK][0m ✓ prometheus-obs-identity-sau-main-dev.service is ACTIVE
[INFO] Validating Prometheus configuration...
[1;32m[OK][0m ✓ Prometheus configuration is VALID
[INFO] Checking Prometheus targets (waiting 35s for first scrape cycle)...
[2026-01-02 06:52:17 UTC] USER=www-data EUID=0 PID=1585605 ACTION=passthru ARGS=grep -q tls_server_config /etc/prometheus/obs-identity-sau-main-dev/web-config.yml
[1;32m[OK][0m ✓ Prometheus has Elasticsearch target configured
[1;32m[OK][0m ✓ Elasticsearch target is UP and being scraped
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] ✅ All monitoring integration steps completed
[INFO] ✅ All verifications PASSED
[INFO] ✅ Elasticsearch registered to dashboard database
[INFO] ✅ Prometheus scraping Elasticsearch metrics
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[1m✓ Step 4 completed successfully![0m
[0;36m[1m════════════════════════════════════════════════════════════════[0m
[0;32m[1m🎉 All deployment tasks completed successfully![0m
[0;32m✓[0m ✅ Search infrastructure (elasticsearch) setup completed successfully
[0;34m[INFO][0m Using eventbus engine from EVENTBUS_ENGINE environment variable: kafka
[0;34m[INFO][0m Cleaning up any existing locks...
[0;32m[1mStarting eventbus engine: kafka[0m
[1;33m═══════════════════════════════════════════════[0m
[0;34m[INFO][0m Loaded from topology.json: identity-sau-main-dev
[0;32m[2026-01-02 06:52:18][0m Loaded environment: identity-sau-main-dev
[0;32m[2026-01-02 06:52:18][0m Service: identity, Zone: sau, Branch: main, Env: dev
[0;32m[2026-01-02 06:52:18][0m VM IP: 142.93.238.16, Interface: eth0:16
[0;32m[2026-01-02 06:52:18][0m Elasticsearch Nodes: 1, PostgreSQL Workers: 1
[0;32m[2026-01-02 06:52:18][0m PostgreSQL HA Nodes: 1, Citus Enabled: yes
[0;32m✓[0m Environment initialized successfully (mode: general)
[0;34m[INFO][0m Starting Kafka setup process...
[0;34m[INFO][0m Steps directory: /opt/fastorder/bash/scripts/env_app_setup/setup/04-eventbus/engine/kafka/steps
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Found 9 step(s) to execute
[0;34m[INFO][0m 📦 Step 1/9: install debezium connector...
═══════════════════════════════════════════════════════════════════
Fetching latest versions from Maven Central...
Installing Debezium PostgreSQL Connector
Debezium version: 3.4.0.Final
pgjdbc version: 42.7.8
═══════════════════════════════════════════════════════════════════
[OK] Debezium 3.4.0.Final with pgjdbc 42.7.8 already installed
[0;32m[OK][0m ✅ Step 1 completed: 00-install-debezium-connector.sh
[0;34m[INFO][0m 📦 Step 2/9: kafka setup...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
🔑 Configuring AWS credentials for secrets vault...
✅ Using permanent AWS credentials from /home/ab/.aws/credentials
🧹 Checking for orphaned Kafka processes on ports 9092, 9093, 8083...
⚠️ Found process on port 9092 (PIDs: [2026-01-02 06:52:19 UTC] USER=www-data EUID=0 PID=1585750 ACTION=passthru ARGS=bash -c lsof -ti tcp:9092 2>/dev/null || true), killing...
[2026-01-02 06:52:19 UTC] USER=www-data EUID=0 PID=1585761 ACTION=passthru ARGS=bash -c kill -9 [2026-01-02 06:52:19 UTC] USER=www-data EUID=0 PID=1585750 ACTION=passthru ARGS=bash -c lsof -ti tcp:9092 2>/dev/null || true 2>/dev/null || true
⚠️ Found process on port 9093 (PIDs: [2026-01-02 06:52:20 UTC] USER=www-data EUID=0 PID=1585777 ACTION=passthru ARGS=bash -c lsof -ti tcp:9093 2>/dev/null || true
1383672
1441337
1559779
1562002
1562365
1562460
1564318
1566063
1566532
1576496
1580069
1580501
1581819), killing...
[2026-01-02 06:52:20 UTC] USER=www-data EUID=0 PID=1585790 ACTION=passthru ARGS=bash -c kill -9 [2026-01-02 06:52:20 UTC] USER=www-data EUID=0 PID=1585777 ACTION=passthru ARGS=bash -c lsof -ti tcp:9093 2>/dev/null || true
1383672
1441337
1559779
1562002
1562365
1562460
1564318
1566063
1566532
1576496
1580069
1580501
1581819 2>/dev/null || true
/usr/bin/bash: line 2: 1383672: command not found
/usr/bin/bash: line 3: 1441337: command not found
/usr/bin/bash: line 4: 1559779: command not found
/usr/bin/bash: line 5: 1562002: command not found
/usr/bin/bash: line 6: 1562365: command not found
/usr/bin/bash: line 7: 1562460: command not found
/usr/bin/bash: line 8: 1564318: command not found
/usr/bin/bash: line 9: 1566063: command not found
/usr/bin/bash: line 10: 1566532: command not found
/usr/bin/bash: line 11: 1576496: command not found
/usr/bin/bash: line 12: 1580069: command not found
/usr/bin/bash: line 13: 1580501: command not found
⚠️ Found process on port 8083 (PIDs: [2026-01-02 06:52:21 UTC] USER=www-data EUID=0 PID=1585821 ACTION=passthru ARGS=bash -c lsof -ti tcp:8083 2>/dev/null || true), killing...
[2026-01-02 06:52:22 UTC] USER=www-data EUID=0 PID=1585833 ACTION=passthru ARGS=bash -c kill -9 [2026-01-02 06:52:21 UTC] USER=www-data EUID=0 PID=1585821 ACTION=passthru ARGS=bash -c lsof -ti tcp:8083 2>/dev/null || true 2>/dev/null || true
✅ Port cleanup completed
Ensuring KAFKA application environment for coordinator...
[0;34m[INFO][0m Creating KAFKA application environment...
[INFO] 🎯 Custom Environment Creation (Example Wrapper)
[INFO] 📁 Orchestrator Library: /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator
[INFO] 💾 State Directory: /opt/fastorder/bash/scripts/env_app_setup/state
[INFO] 🚀 Calling centralized orchestrator: fo-env create-app
[INFO] 📋 Arguments: --service identity --zone sau --branch main --env dev --domain eventbus-identity-sau-main-dev-kafka-connect --app kafka-connect
[INFO] Creating application-specific environment configuration
[INFO] Environment ID: identity-sau-main-dev
[INFO] Application: kafka-connect
[INFO] Base environment identity-sau-main-dev already exists
[INFO] Allocated kafka-connect IP: 10.100.1.211
[INFO] Generated domain: eventbus-identity-sau-main-dev-kafka-connect.fastorder.com
[INFO] Configuring network interface for kafka-connect IP: 10.100.1.211
[2026-01-02 06:52:24 UTC] USER=www-data EUID=0 PID=1586241 ACTION=passthru ARGS=ip addr add 10.100.1.211/32 dev eth0 label eth0:211
[ OK ] Configured kafka-connect IP 10.100.1.211 on interface eth0:211
[INFO] Creating systemd service for kafka-connect IP persistence...
[2026-01-02 06:52:24 UTC] USER=www-data EUID=0 PID=1586260 ACTION=passthru ARGS=systemctl daemon-reload
[ OK ] kafka-connect IP will persist across reboots
[INFO] Updating topology with application-specific configuration...
[ OK ] Topology updated with application-specific configuration
[INFO] Binding kafka-connect IP to domain: 10.100.1.211 -> eventbus-identity-sau-main-dev-kafka-connect.fastorder.com
[ OK ] Successfully bound eventbus-identity-sau-main-dev-kafka-connect.fastorder.com to 10.100.1.211
[ OK ] Domain correctly mapped
[ OK ] Application environment created successfully!
[INFO]
[INFO] Application Details:
[INFO] Environment ID: identity-sau-main-dev
[INFO] Application: kafka-connect
[INFO] IP: 10.100.1.211
[INFO] Domain: eventbus-identity-sau-main-dev-kafka-connect.fastorder.com
[INFO]
[INFO] To use this application:
[INFO] source /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh
[INFO] init_environment kafka-connect
[INFO] echo $VM_IP # Returns: 10.100.1.211
[ OK ] 🎉 Environment creation completed successfully!
[INFO] 📋 What happened:
[INFO] ✅ Called centralized orchestrator at /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator
[INFO] ✅ All topology.json management handled centrally
[INFO] ✅ Application-specific IP and domain configured
[INFO] ✅ Network interface configured and made persistent
[INFO] ✅ Domain binding added to /etc/hosts (if not skipped)
[INFO] 🔧 To use the centralized orchestrator directly:
[INFO] # Add orchestrator to PATH
[INFO] export PATH="/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/bin:$PATH"
[INFO] # Then call directly
[INFO] fo-env create-app --service auth --zone uae --env dev --app redis
[INFO] 📚 For more orchestrator commands:
[INFO] fo-env --help
[0;32mCreated KAFKA environment: eventbus-identity-sau-main-dev-kafka-connect.fastorder.com (10.100.1.211)[0m
Ensuring KAFKA_BROKER_IP application environment for coordinator...
[0;34m[INFO][0m Creating KAFKA application environment...
[INFO] 🎯 Custom Environment Creation (Example Wrapper)
[INFO] 📁 Orchestrator Library: /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator
[INFO] 💾 State Directory: /opt/fastorder/bash/scripts/env_app_setup/state
[INFO] 🚀 Calling centralized orchestrator: fo-env create-app
[INFO] 📋 Arguments: --service identity --zone sau --branch main --env dev --domain eventbus-identity-sau-main-dev-kafka-broker-01 --app kafka-broker
[INFO] Creating application-specific environment configuration
[INFO] Environment ID: identity-sau-main-dev
[INFO] Application: kafka-broker
[INFO] Base environment identity-sau-main-dev already exists
[INFO] Allocated kafka-broker IP: 10.100.1.212
[INFO] Generated domain: eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com
[INFO] Configuring network interface for kafka-broker IP: 10.100.1.212
[2026-01-02 06:52:27 UTC] USER=www-data EUID=0 PID=1586799 ACTION=passthru ARGS=ip addr add 10.100.1.212/32 dev eth0 label eth0:212
[ OK ] Configured kafka-broker IP 10.100.1.212 on interface eth0:212
[INFO] Creating systemd service for kafka-broker IP persistence...
[2026-01-02 06:52:27 UTC] USER=www-data EUID=0 PID=1586818 ACTION=passthru ARGS=systemctl daemon-reload
[ OK ] kafka-broker IP will persist across reboots
[INFO] Updating topology with application-specific configuration...
[ OK ] Topology updated with application-specific configuration
[INFO] Binding kafka-broker IP to domain: 10.100.1.212 -> eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com
[ OK ] Successfully bound eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com to 10.100.1.212
[ OK ] Domain correctly mapped
[ OK ] Application environment created successfully!
[INFO]
[INFO] Application Details:
[INFO] Environment ID: identity-sau-main-dev
[INFO] Application: kafka-broker
[INFO] IP: 10.100.1.212
[INFO] Domain: eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com
[INFO]
[INFO] To use this application:
[INFO] source /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh
[INFO] init_environment kafka-broker
[INFO] echo $VM_IP # Returns: 10.100.1.212
[ OK ] 🎉 Environment creation completed successfully!
[INFO] 📋 What happened:
[INFO] ✅ Called centralized orchestrator at /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator
[INFO] ✅ All topology.json management handled centrally
[INFO] ✅ Application-specific IP and domain configured
[INFO] ✅ Network interface configured and made persistent
[INFO] ✅ Domain binding added to /etc/hosts (if not skipped)
[INFO] 🔧 To use the centralized orchestrator directly:
[INFO] # Add orchestrator to PATH
[INFO] export PATH="/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/bin:$PATH"
[INFO] # Then call directly
[INFO] fo-env create-app --service auth --zone uae --env dev --app redis
[INFO] 📚 For more orchestrator commands:
[INFO] fo-env --help
[0;32mCreated KAFKA_BROKER_DOMAIN environment: eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com (10.100.1.212)[0m
[0;34m[INFO][0m Kafka Broker IP: 10.100.1.212
[0;34m[INFO][0m Kafka Connect IP: 10.100.1.211
[0;34m[INFO][0m Registered /etc/hosts: eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com -> 10.100.1.212
[0;34m[INFO][0m Registered /etc/hosts: eventbus-identity-sau-main-dev-kafka-connect.fastorder.com -> 10.100.1.211
🔐 Initializing keystore passwords...
[0;34m[INFO][0m 🔍 Checking secrets backend (provider: aws)...
[0;33m⚠️ No existing passwords found[0m
[0;34m[INFO][0m 🔐 Generating new random passwords...
[0;32m✅ Generated new passwords[0m
[0;34m[INFO][0m 🔐 Vaulting kafka passwords to remote backend...
[0;32m✅ Passwords vaulted to remote backend[0m
✅ Keystore passwords initialized
- Keystore password: E4FDSwWT... (32 chars)
- Truststore password: yOb0eqkA... (32 chars)
[0;34m[INFO][0m 🔐 Vaulting kafka passwords to remote backend...
[0;32m✅ Passwords vaulted to remote backend[0m
✅ Kafka keystore passwords saved to AWS Secrets Manager
[INFO] Generating for: identity-sau-main-dev (host=eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com ip=10.100.1.212)
[2026-01-02 06:52:42 UTC] USER=www-data EUID=0 PID=1587295 ACTION=fsop ARGS=rm -rf /opt/kafka/secrets/identity-sau-main-dev/coordinator /var/lib/kafka/identity-sau-main-dev/coordinator
[2026-01-02 06:52:42 UTC] USER=www-data EUID=0 PID=1587304 ACTION=fsop ARGS=mkdir -p /opt/kafka/secrets/identity-sau-main-dev/coordinator /opt/kafka/config/identity-sau-main-dev/coordinator /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem /var/lib/kafka/identity-sau-main-dev_coordinator-data
[2026-01-02 06:52:42 UTC] USER=www-data EUID=0 PID=1587322 ACTION=fsop ARGS=chown -R kafka:kafka /opt/kafka/config/identity-sau-main-dev/coordinator /var/lib/kafka/identity-sau-main-dev_coordinator-data
[2026-01-02 06:52:42 UTC] USER=www-data EUID=0 PID=1587331 ACTION=fsop ARGS=chmod 770 /opt/kafka/config/identity-sau-main-dev/coordinator /var/lib/kafka/identity-sau-main-dev_coordinator-data
[2026-01-02 06:52:42 UTC] USER=www-data EUID=0 PID=1587340 ACTION=fsop ARGS=chmod 750 /opt/kafka/secrets/identity-sau-main-dev/coordinator
[2026-01-02 06:52:42 UTC] USER=www-data EUID=0 PID=1587349 ACTION=fsop ARGS=chmod 750 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
[2026-01-02 06:52:42 UTC] USER=www-data EUID=0 PID=1587359 ACTION=fsop ARGS=chmod 700 /tmp/fo-tls.GfCIDF
[2026-01-02 06:52:43 UTC] USER=www-data EUID=0 PID=1587369 ACTION=fsop ARGS=chmod 755 /tmp/fo-tls.GfCIDF
[2026-01-02 06:52:43 UTC] USER=www-data EUID=0 PID=1587379 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/fo-tls.GfCIDF/ra_root.crt
[2026-01-02 06:52:43 UTC] USER=www-data EUID=0 PID=1587391 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/fo-tls.GfCIDF/ra_root.key
[2026-01-02 06:52:43 UTC] USER=www-data EUID=0 PID=1587401 ACTION=fsop ARGS=chmod 644 /tmp/fo-tls.GfCIDF/ra_root.crt
[2026-01-02 06:52:43 UTC] USER=www-data EUID=0 PID=1587411 ACTION=fsop ARGS=chmod 644 /tmp/fo-tls.GfCIDF/ra_root.key
Certificate was added to keystore
[2026-01-02 06:52:44 UTC] USER=www-data EUID=0 PID=1587446 ACTION=fsop ARGS=mv /tmp/fo-tls.GfCIDF/truststore.jks /opt/kafka/secrets/identity-sau-main-dev/coordinator/truststore.jks
[2026-01-02 06:52:44 UTC] USER=www-data EUID=0 PID=1587455 ACTION=fsop ARGS=chown kafka:kafka /opt/kafka/secrets/identity-sau-main-dev/coordinator/truststore.jks
[2026-01-02 06:52:44 UTC] USER=www-data EUID=0 PID=1587464 ACTION=fsop ARGS=chmod 0640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/truststore.jks
Generating 4,096 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 825 days
for: CN=eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com, OU=Kafka Broker, O=FastOrder, C=AE
Warning:
The JKS keystore uses a proprietary format. It is recommended to migrate to PKCS12 which is an industry standard format using "keytool -importkeystore -srckeystore /tmp/fo-tls.GfCIDF/kafka.server.keystore.jks -destkeystore /tmp/fo-tls.GfCIDF/kafka.server.keystore.jks -deststoretype pkcs12".
Warning:
The JKS keystore uses a proprietary format. It is recommended to migrate to PKCS12 which is an industry standard format using "keytool -importkeystore -srckeystore /tmp/fo-tls.GfCIDF/kafka.server.keystore.jks -destkeystore /tmp/fo-tls.GfCIDF/kafka.server.keystore.jks -deststoretype pkcs12".
Certificate request self-signature ok
subject=C = AE, O = FastOrder, OU = Kafka Broker, CN = eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com
Certificate was added to keystore
Warning:
Certificate reply was installed in keystore
Warning:
[2026-01-02 06:52:48 UTC] USER=www-data EUID=0 PID=1587693 ACTION=fsop ARGS=mv /tmp/fo-tls.GfCIDF/kafka.server.keystore.jks /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.server.keystore.jks
[2026-01-02 06:52:48 UTC] USER=www-data EUID=0 PID=1587702 ACTION=fsop ARGS=chown kafka:kafka /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.server.keystore.jks
[2026-01-02 06:52:48 UTC] USER=www-data EUID=0 PID=1587711 ACTION=fsop ARGS=chmod 0640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.server.keystore.jks
Generating 4,096 bit RSA key pair and self-signed certificate (SHA384withRSA) with a validity of 825 days
for: CN=eventbus-identity-sau-main-dev-kafka-connect.fastorder.com, OU=Kafka Connect REST, O=FastOrder, C=AE
Certificate request self-signature ok
subject=C = AE, O = FastOrder, OU = Kafka Connect REST, CN = eventbus-identity-sau-main-dev-kafka-connect.fastorder.com
Certificate was added to keystore
Certificate reply was installed in keystore
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587892 ACTION=fsop ARGS=mv /tmp/fo-tls.GfCIDF/connect-rest.keystore.p12 /opt/kafka/secrets/identity-sau-main-dev/coordinator/connect-rest.keystore.p12
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587901 ACTION=fsop ARGS=chown kafka:kafka /opt/kafka/secrets/identity-sau-main-dev/coordinator/connect-rest.keystore.p12
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587910 ACTION=fsop ARGS=chmod 0640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/connect-rest.keystore.p12
Certificate request self-signature ok
subject=CN = kafka-client-identity-sau-main-dev, OU = Kafka Client, O = FastOrder, C = AE
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587922 ACTION=fsop ARGS=cp /tmp/fo-tls.GfCIDF/ra_root.crt /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587931 ACTION=fsop ARGS=cp /tmp/fo-tls.GfCIDF/client-key.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587940 ACTION=fsop ARGS=cp /tmp/fo-tls.GfCIDF/client-cert.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587949 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587958 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587967 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587983 ACTION=fsop ARGS=mv /tmp/fo-tls.GfCIDF/kafka.client.keystore.p12 /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.client.keystore.p12
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1587992 ACTION=fsop ARGS=chown kafka:kafka /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.client.keystore.p12
[2026-01-02 06:52:57 UTC] USER=www-data EUID=0 PID=1588001 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.client.keystore.p12
🔐 Ensuring kafka user has access to PostgreSQL certificates...
✅ kafka is already in postgres group
🧹 Cleaning up conflicting services and processes on Kafka ports on 10.100.1.212...
🔪 Killing processes on 10.100.1.212:8083: [2026-01-02 06:52:58 UTC] USER=www-data EUID=0 PID=1588039 ACTION=passthru ARGS=bash -c lsof -ti tcp:8083 -sTCP:LISTEN 2>/dev/null | xargs -I {} lsof -p {} -a -i @10.100.1.212:8083 -t 2>/dev/null || true
[2026-01-02 06:52:58 UTC] USER=www-data EUID=0 PID=1588056 ACTION=passthru ARGS=bash -c kill -9 [2026-01-02 06:52:58 UTC] USER=www-data EUID=0 PID=1588039 ACTION=passthru ARGS=bash -c lsof -ti tcp:8083 -sTCP:LISTEN 2>/dev/null | xargs -I {} lsof -p {} -a -i @10.100.1.212:8083 -t 2>/dev/null || true
🔪 Killing processes on 10.100.1.212:9092: [2026-01-02 06:52:58 UTC] USER=www-data EUID=0 PID=1588067 ACTION=passthru ARGS=bash -c lsof -ti tcp:9092 -sTCP:LISTEN 2>/dev/null | xargs -I {} lsof -p {} -a -i @10.100.1.212:9092 -t 2>/dev/null || true
[2026-01-02 06:52:58 UTC] USER=www-data EUID=0 PID=1588080 ACTION=passthru ARGS=bash -c kill -9 [2026-01-02 06:52:58 UTC] USER=www-data EUID=0 PID=1588067 ACTION=passthru ARGS=bash -c lsof -ti tcp:9092 -sTCP:LISTEN 2>/dev/null | xargs -I {} lsof -p {} -a -i @10.100.1.212:9092 -t 2>/dev/null || true
🔪 Killing processes on 10.100.1.212:9093: [2026-01-02 06:52:58 UTC] USER=www-data EUID=0 PID=1588091 ACTION=passthru ARGS=bash -c lsof -ti tcp:9093 -sTCP:LISTEN 2>/dev/null | xargs -I {} lsof -p {} -a -i @10.100.1.212:9093 -t 2>/dev/null || true
[2026-01-02 06:52:59 UTC] USER=www-data EUID=0 PID=1588110 ACTION=passthru ARGS=bash -c kill -9 [2026-01-02 06:52:58 UTC] USER=www-data EUID=0 PID=1588091 ACTION=passthru ARGS=bash -c lsof -ti tcp:9093 -sTCP:LISTEN 2>/dev/null | xargs -I {} lsof -p {} -a -i @10.100.1.212:9093 -t 2>/dev/null || true
✅ Port cleanup completed
🔧 Checking for Kafka Connect internal topics with incorrect cleanup policy...
ℹ️ Kafka broker not running, skipping topic cleanup policy fix
🔧 Creating environment-specific systemd units...
🔧 Writing client properties to /etc/kafka/client-identity-sau-main-dev-coordinator.properties ...
[2026-01-02 06:52:59 UTC] USER=www-data EUID=0 PID=1588149 ACTION=fsop ARGS=chown root:kafka /etc/kafka/client-identity-sau-main-dev-coordinator.properties
[2026-01-02 06:52:59 UTC] USER=www-data EUID=0 PID=1588158 ACTION=fsop ARGS=chmod 0640 /etc/kafka/client-identity-sau-main-dev-coordinator.properties
[2026-01-02 06:52:59 UTC] USER=www-data EUID=0 PID=1588167 ACTION=passthru ARGS=systemctl daemon-reload
Failed to print table: Broken pipe
Failed to print table: Broken pipe
🔒 Adjusting group ownership and permissions ...
[2026-01-02 06:53:09 UTC] USER=www-data EUID=0 PID=1588319 ACTION=fsop ARGS=chown :kafka /opt/kafka/secrets/identity-sau-main-dev/coordinator/truststore.jks /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.server.keystore.jks
[2026-01-02 06:53:09 UTC] USER=www-data EUID=0 PID=1588328 ACTION=fsop ARGS=chmod 0640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/truststore.jks /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.server.keystore.jks
[2026-01-02 06:53:09 UTC] USER=www-data EUID=0 PID=1588337 ACTION=fsop ARGS=chmod 0640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.client.keystore.p12
[2026-01-02 06:53:09 UTC] USER=www-data EUID=0 PID=1588347 ACTION=fsop ARGS=chown root:kafka /etc/kafka/client-identity-sau-main-dev-coordinator.properties
[2026-01-02 06:53:09 UTC] USER=www-data EUID=0 PID=1588358 ACTION=fsop ARGS=chmod 0640 /etc/kafka/client-identity-sau-main-dev-coordinator.properties
✅ Kafka configuration complete for identity-sau-main-dev_coordinator
Broker ID : 5
Broker keystore : /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.server.keystore.jks
REST keystore : /opt/kafka/secrets/identity-sau-main-dev/coordinator/connect-rest.keystore.p12
Truststore : /opt/kafka/secrets/identity-sau-main-dev/coordinator/truststore.jks
Client PKCS12 : /opt/kafka/secrets/identity-sau-main-dev/coordinator/kafka.client.keystore.p12
Data directory : /var/lib/kafka/identity-sau-main-dev_coordinator-data
Server config : /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
Connect config : /opt/kafka/config/identity-sau-main-dev/coordinator/connect-distributed.properties
CLI client config : /etc/kafka/client-identity-sau-main-dev-coordinator.properties
🎯 Next step: Run 03-restart-kafka-related-services.sh to start services
[0;32m[OK][0m ✅ Step 2 completed: 01-kafka-setup.sh
[0;34m[INFO][0m 📦 Step 3/9: metadata...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] 🎯 Kafka metadata mode: kraft
╔════════════════════════════════════════════════════════════════════╗
║ Kafka Metadata Layer Setup ║
╚════════════════════════════════════════════════════════════════════╝
Environment : identity-sau-main-dev
Service : identity
Zone : sau
Branch : main
Environment : dev
VM IP : 142.93.238.16
Metadata Mode : kraft
📋 KRaft Mode (Modern)
────────────────────────────────────────────────────────────────
✅ No ZooKeeper dependency
✅ Faster metadata operations
✅ Simplified architecture
✅ Recommended for new deployments
⚠️ Requires Kafka 3.3+ in production
════════════════════════════════════════════════════════════════════
[INFO] 🚀 Executing KRaft setup script...
[INFO] Script: /opt/fastorder/bash/scripts/env_app_setup/setup/04-eventbus/engine/kafka/steps/metadata/kraft.sh
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[2026-01-02 06:53:10 UTC] USER=www-data EUID=0 PID=1588418 ACTION=fsop ARGS=mkdir -p /var/lib/kafka/identity-sau-main-dev_coordinator-meta /opt/kafka/config/identity-sau-main-dev/coordinator /var/lib/kafka/identity-sau-main-dev_coordinator-data
[2026-01-02 06:53:10 UTC] USER=www-data EUID=0 PID=1588427 ACTION=fsop ARGS=chown -R kafka:kafka /var/lib/kafka/identity-sau-main-dev_coordinator-meta /opt/kafka/config/identity-sau-main-dev/coordinator /var/lib/kafka/identity-sau-main-dev_coordinator-data
[2026-01-02 06:53:10 UTC] USER=www-data EUID=0 PID=1588436 ACTION=fsop ARGS=chmod 770 /var/lib/kafka/identity-sau-main-dev_coordinator-meta /opt/kafka/config/identity-sau-main-dev/coordinator /var/lib/kafka/identity-sau-main-dev_coordinator-data
[INFO] Adding eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com to /etc/hosts -> 10.100.1.212
[INFO] Adding eventbus-identity-sau-main-dev-kafka-connect.fastorder.com to /etc/hosts -> 10.100.1.211
[INFO] Setting up KRaft for: identity-sau-main-dev (host=eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com ip=10.100.1.212)
[2026-01-02 06:53:10 UTC] USER=www-data EUID=0 PID=1588460 ACTION=fsop ARGS=mkdir -p /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev
[INFO] Generated cluster.id=zSMUIjftQt6oa0Ax-9uqsA
🔧 Configuring Kafka for KRaft mode...
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589027 ACTION=fsop ARGS=test -f /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589041 ACTION=fsop ARGS=test -r /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589050 ACTION=fsop ARGS=sed -i /^zookeeper\.connect=/d /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589059 ACTION=passthru ARGS=bash -c grep -q '^process.roles=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589079 ACTION=passthru ARGS=bash -c grep -q '^node.id=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589099 ACTION=passthru ARGS=bash -c grep -q '^broker.id=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589108 ACTION=fsop ARGS=sed -i s|^broker.id=.*|broker.id=1| /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589140 ACTION=passthru ARGS=bash -c grep -q '^controller.quorum.voters=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589159 ACTION=passthru ARGS=bash -c grep -q '^metadata.log.dir=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589179 ACTION=passthru ARGS=bash -c grep -q '^log.dirs=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589189 ACTION=fsop ARGS=sed -i s|^log.dirs=.*|log.dirs=/var/lib/kafka/identity-sau-main-dev_coordinator-data| /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589198 ACTION=passthru ARGS=bash -c grep -q '^listeners=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589208 ACTION=passthru ARGS=bash -c grep -q 'CONTROLLER://' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:14 UTC] USER=www-data EUID=0 PID=1589217 ACTION=fsop ARGS=sed -i s|^listeners=.*|listeners=SSL://10.100.1.212:9092,CONTROLLER://10.100.1.212:9093| /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589227 ACTION=passthru ARGS=bash -c grep -q '^advertised.listeners=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589236 ACTION=fsop ARGS=sed -i s|^advertised.listeners=.*|advertised.listeners=SSL://eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092| /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589245 ACTION=passthru ARGS=bash -c grep -q '^listener.security.protocol.map=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589254 ACTION=fsop ARGS=sed -i s|^listener.security.protocol.map=.*|listener.security.protocol.map=SSL:SSL,CONTROLLER:PLAINTEXT| /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589263 ACTION=passthru ARGS=bash -c grep -q '^inter.broker.listener.name=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589272 ACTION=fsop ARGS=sed -i s|^inter.broker.listener.name=.*|inter.broker.listener.name=SSL| /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589281 ACTION=passthru ARGS=bash -c grep -q '^offsets.topic.replication.factor=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589290 ACTION=fsop ARGS=sed -i s|^offsets.topic.replication.factor=.*|offsets.topic.replication.factor=1| /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589300 ACTION=passthru ARGS=bash -c grep -q '^transaction.state.log.replication.factor=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589309 ACTION=fsop ARGS=sed -i s|^transaction.state.log.replication.factor=.*|transaction.state.log.replication.factor=1| /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589318 ACTION=passthru ARGS=bash -c grep -q '^transaction.state.log.min.isr=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589327 ACTION=fsop ARGS=sed -i s|^transaction.state.log.min.isr=.*|transaction.state.log.min.isr=1| /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589336 ACTION=passthru ARGS=bash -c grep -q '^min.insync.replicas=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
✅ KRaft configuration applied to server.properties
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589355 ACTION=fsop ARGS=test -f /var/lib/kafka/identity-sau-main-dev_coordinator-meta/meta.properties
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589364 ACTION=fsop ARGS=test -f /var/lib/kafka/identity-sau-main-dev_coordinator-data/meta.properties
[INFO] Already formatted: both /var/lib/kafka/identity-sau-main-dev_coordinator-meta and /var/lib/kafka/identity-sau-main-dev_coordinator-data have meta.properties
🔧 Creating/refreshing KRaft systemd unit...
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589382 ACTION=fsop ARGS=sed -i s|\\$MAINPID|$MAINPID|g /etc/systemd/system/confluent-kraft-identity-sau-main-dev_coordinator.service
[2026-01-02 06:53:15 UTC] USER=www-data EUID=0 PID=1589391 ACTION=passthru ARGS=systemctl daemon-reload
✅ Ensured confluent-kraft-identity-sau-main-dev_coordinator.service
🛑 Stopping legacy ZooKeeper-mode services and current KRaft instance...
🧹 Cleaning up rogue Kafka processes...
🧹 Killing any processes holding Kafka ports 9092, 9093...
🔪 Killing processes on port 9093: 1383672
1441337
1559779
1562002
1562365
1564318
1566063
1576496
1580069
1580501
1581819
[2026-01-02 06:53:16 UTC] USER=www-data EUID=0 PID=1589483 ACTION=passthru ARGS=bash -c kill -9 1383672
[2026-01-02 06:53:16 UTC] USER=www-data EUID=0 PID=1589494 ACTION=passthru ARGS=bash -c kill -9 1441337
[2026-01-02 06:53:16 UTC] USER=www-data EUID=0 PID=1589506 ACTION=passthru ARGS=bash -c kill -9 1559779
[2026-01-02 06:53:16 UTC] USER=www-data EUID=0 PID=1589517 ACTION=passthru ARGS=bash -c kill -9 1562002
[2026-01-02 06:53:16 UTC] USER=www-data EUID=0 PID=1589526 ACTION=passthru ARGS=bash -c kill -9 1562365
[2026-01-02 06:53:17 UTC] USER=www-data EUID=0 PID=1589535 ACTION=passthru ARGS=bash -c kill -9 1564318
[2026-01-02 06:53:17 UTC] USER=www-data EUID=0 PID=1589544 ACTION=passthru ARGS=bash -c kill -9 1566063
[2026-01-02 06:53:17 UTC] USER=www-data EUID=0 PID=1589555 ACTION=passthru ARGS=bash -c kill -9 1576496
[2026-01-02 06:53:17 UTC] USER=www-data EUID=0 PID=1589564 ACTION=passthru ARGS=bash -c kill -9 1580069
[2026-01-02 06:53:17 UTC] USER=www-data EUID=0 PID=1589573 ACTION=passthru ARGS=bash -c kill -9 1580501
[2026-01-02 06:53:17 UTC] USER=www-data EUID=0 PID=1589582 ACTION=passthru ARGS=bash -c kill -9 1581819
✅ Legacy services stopped and rogue processes cleaned
🔓 Removing stale lock files...
[2026-01-02 06:53:22 UTC] USER=www-data EUID=0 PID=1589693 ACTION=fsop ARGS=test -f /var/lib/kafka/identity-sau-main-dev_coordinator-meta/.lock
🗑️ Removing stale lock: /var/lib/kafka/identity-sau-main-dev_coordinator-meta/.lock
[2026-01-02 06:53:22 UTC] USER=www-data EUID=0 PID=1589702 ACTION=fsop ARGS=rm -f /var/lib/kafka/identity-sau-main-dev_coordinator-meta/.lock
[2026-01-02 06:53:22 UTC] USER=www-data EUID=0 PID=1589712 ACTION=fsop ARGS=test -f /var/lib/kafka/identity-sau-main-dev_coordinator-data/.lock
🗑️ Removing stale lock: /var/lib/kafka/identity-sau-main-dev_coordinator-data/.lock
[2026-01-02 06:53:22 UTC] USER=www-data EUID=0 PID=1589722 ACTION=fsop ARGS=rm -f /var/lib/kafka/identity-sau-main-dev_coordinator-data/.lock
✅ Lock file check complete
🚀 Starting confluent-kraft-identity-sau-main-dev_coordinator.service ...
[2026-01-02 06:53:22 UTC] USER=www-data EUID=0 PID=1589735 ACTION=passthru ARGS=systemctl enable confluent-kraft-identity-sau-main-dev_coordinator.service
Created symlink /etc/systemd/system/multi-user.target.wants/confluent-kraft-identity-sau-main-dev_coordinator.service -> /etc/systemd/system/confluent-kraft-identity-sau-main-dev_coordinator.service.
[2026-01-02 06:53:23 UTC] USER=www-data EUID=0 PID=1589783 ACTION=passthru ARGS=systemctl restart confluent-kraft-identity-sau-main-dev_coordinator.service
🔧 Patching shared Connect unit to follow KRaft broker...
[2026-01-02 06:53:26 UTC] USER=www-data EUID=0 PID=1590238 ACTION=fsop ARGS=sed -i -e s|${FULL_ENV}|identity-sau-main-dev|g -e s|${IDENTIFIER}|coordinator|g -e s|${CONFIG_DIR}|/opt/kafka/config/identity-sau-main-dev/coordinator|g /etc/systemd/system/confluent-connect-identity-sau-main-dev_coordinator.service
[2026-01-02 06:53:26 UTC] USER=www-data EUID=0 PID=1590247 ACTION=fsop ARGS=sed -i s|\\$MAINPID|$MAINPID|g /etc/systemd/system/confluent-connect-identity-sau-main-dev_coordinator.service
[2026-01-02 06:53:26 UTC] USER=www-data EUID=0 PID=1590257 ACTION=fsop ARGS=sed -i s|^After=.*|After=network-online.target confluent-kraft-identity-sau-main-dev_coordinator.service| /etc/systemd/system/confluent-connect-identity-sau-main-dev_coordinator.service
[2026-01-02 06:53:26 UTC] USER=www-data EUID=0 PID=1590269 ACTION=fsop ARGS=sed -i s|^Wants=.*|Wants=confluent-kraft-identity-sau-main-dev_coordinator.service| /etc/systemd/system/confluent-connect-identity-sau-main-dev_coordinator.service
[2026-01-02 06:53:26 UTC] USER=www-data EUID=0 PID=1590281 ACTION=fsop ARGS=sed -i s|^ExecStart=.*|ExecStart=/opt/kafka/bin/connect-distributed.sh /opt/kafka/config/identity-sau-main-dev/coordinator/connect-distributed.properties| /etc/systemd/system/confluent-connect-identity-sau-main-dev_coordinator.service
[2026-01-02 06:53:26 UTC] USER=www-data EUID=0 PID=1590293 ACTION=passthru ARGS=systemctl daemon-reload
✅ Connect unit patched
[2026-01-02 06:53:27 UTC] USER=www-data EUID=0 PID=1590368 ACTION=fsop ARGS=ln -sf /opt/kafka/config/identity-sau-main-dev/coordinator/connect-distributed.properties /opt/kafka/config/connect-distributed.properties
⏳ Waiting for broker coordinator on SSL://eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092 ...
⏳ Waiting for KRaft broker... (attempt 1, 0s/600s)
Debug: Last error was: [2026-01-02 06:53:27 UTC] USER=www-data EUID=0 PID=1590395 ACTION=passthru ARGS=bash -c timeout 5 sudo -u kafka /opt/kafka/bin/kafka-metadata-quorum.sh --bootstrap-server 'eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092' --command-config '/etc/kafka/client-identity-sau-main-dev-coordinator.properties' describe --status
[2026-01-02 06:53:31,215] WARN [AdminClient clientId=adminclient-1] Connection to node -1 (eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com/10.100.1.212:9092) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient)
[2026-01-02 06:53:31,326] WARN [AdminClient clientId=adminclient-1] Connection to node -1 (eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com/10.100.1.212:9092) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient)
[2026-01-02 06:53:31,431] WARN [AdminClient clientId=adminclient-1] Connection to node -1 (eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com/10.100.1.212:9092) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient)
[2026-01-02 06:53:31,637] WARN [AdminClient clientId=adminclient-1] Connection to node -1 (eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com/10.100.1.212:9092) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient)
[2026-01-02 06:53:32,051] WARN [AdminClient clientId=adminclient-1] Connection to node -1 (eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com/10.100.1.212:9092) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient)
[2026-01-02 06:53:32,861] WARN [AdminClient clientId=adminclient-1] Connection to node -1 (eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com/10.100.1.212:9092) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient)
✅ coordinator responded after 13s (attempt 3)
---- server.properties (key lines) ----
[2026-01-02 06:54:06 UTC] USER=www-data EUID=0 PID=1593968 ACTION=passthru ARGS=bash -c grep -E '^(listeners|advertised\.listeners|process\.roles|controller\.quorum\.voters|controller\.listener\.names|inter\.broker\.listener\.name|log\.dirs|metadata\.log\.dir)=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties'
listeners=SSL://10.100.1.212:9092,CONTROLLER://10.100.1.212:9093
advertised.listeners=SSL://eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092
inter.broker.listener.name=SSL
log.dirs=/var/lib/kafka/identity-sau-main-dev_coordinator-data
process.roles=broker,controller
controller.listener.names=CONTROLLER
controller.quorum.voters=1@10.100.1.212:9093
metadata.log.dir=/var/lib/kafka/identity-sau-main-dev_coordinator-meta
---------------------------------------
✅ KRaft setup complete for identity-sau-main-dev_coordinator
server.properties : /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
data dir : /var/lib/kafka/identity-sau-main-dev_coordinator-data
meta dir : /var/lib/kafka/identity-sau-main-dev_coordinator-meta
systemd unit : confluent-kraft-identity-sau-main-dev_coordinator.service
🔧 Kafka Configuration Modified:
✓ process.roles, node.id, controller.quorum.voters, controller.listener.names
✓ listeners (SSL + CONTROLLER) and advertised.listeners (FQDN fallback to IP)
✓ listener.security.protocol.map, inter.broker.listener.name
✓ log.dirs -> /var/lib/kafka/identity-sau-main-dev_coordinator-data, metadata.log.dir -> /var/lib/kafka/identity-sau-main-dev_coordinator-meta
✓ removed zookeeper.connect (if present)
✓ created/refreshed dedicated KRaft systemd unit
✓ patched shared Connect unit to follow KRaft broker
✓ symlinked /opt/kafka/config/identity-sau-main-dev/coordinator/connect-distributed.properties -> /opt/kafka/config/connect-distributed.properties (compat)
🔎 Check quorum:
/opt/kafka/bin/kafka-metadata-quorum.sh --bootstrap-server eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092 --command-config /etc/kafka/client-identity-sau-main-dev-coordinator.properties describe --status
📋 Next steps:
1) Review KRaft config: sudo grep -E 'process.roles|node.id|controller|listeners|advertised.listeners|log.dirs|metadata.log.dir' /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
2) Verify topics: /opt/kafka/bin/kafka-topics.sh --bootstrap-server eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092 --command-config /etc/kafka/client-identity-sau-main-dev-coordinator.properties --list
✅ KRaft metadata layer setup completed successfully
Next steps:
1. Verify KRaft quorum status
2. Create Kafka topics
3. Configure Kafka Connect
[2026-01-02 06:54:06 UTC] USER=www-data EUID=0 PID=1593978 ACTION=fsop ARGS=mkdir -p /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev
[INFO] Saved metadata mode to: /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/kafka_metadata_mode
════════════════════════════════════════════════════════════════════
✅ Kafka Metadata Layer Setup Complete
Mode : kraft
Environment : identity-sau-main-dev
State saved : /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/kafka_metadata_mode
KRaft cluster.id: zSMUIjftQt6oa0Ax-9uqsA
Verify quorum:
kafka-metadata-quorum.sh --bootstrap-server ... describe
════════════════════════════════════════════════════════════════════
[0;32m[OK][0m ✅ Step 3 completed: 02-metadata.sh
[0;34m[INFO][0m 📦 Step 4/9: restart kafka related services...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[2026-01-02 06:54:07 UTC] USER=www-data EUID=0 PID=1594037 ACTION=fsop ARGS=test -f /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:54:07 UTC] USER=www-data EUID=0 PID=1594047 ACTION=passthru ARGS=bash -c grep -E '^[[:space:]]*process\.roles=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties' | grep -Eq '(broker|controller)'
[INFO] 📋 Detected mode from server.properties: kraft
[2026-01-02 06:54:08 UTC] USER=www-data EUID=0 PID=1594127 ACTION=passthru ARGS=systemctl stop confluent-connect-identity-sau-main-dev_coordinator.service
[2026-01-02 06:54:10 UTC] USER=www-data EUID=0 PID=1594256 ACTION=passthru ARGS=systemctl stop confluent-zookeeper-identity-sau-main-dev_coordinator.service
Failed to stop confluent-zookeeper-identity-sau-main-dev_coordinator.service: Unit confluent-zookeeper-identity-sau-main-dev_coordinator.service not loaded.
[INFO] 🧹 Removing stale Kafka lock files...
[2026-01-02 06:54:13 UTC] USER=www-data EUID=0 PID=1594295 ACTION=fsop ARGS=rm -f /var/lib/kafka/identity-sau-main-dev_coordinator-meta/.lock
[2026-01-02 06:54:13 UTC] USER=www-data EUID=0 PID=1594304 ACTION=fsop ARGS=rm -f /var/lib/kafka/identity-sau-main-dev_coordinator-data/.lock
[INFO] 🧹 Cleaning up orphaned processes on Kafka ports...
[2026-01-02 06:54:13 UTC] USER=www-data EUID=0 PID=1594313 ACTION=passthru ARGS=bash -c
for port in 9092 9093 8083 2181; do
pids=$(lsof -ti tcp:$port 2>/dev/null || true)
if [[ -n "$pids" ]]; then
echo " Killing orphaned processes on port $port: $pids"
kill -9 $pids 2>/dev/null || true
sleep 1
fi
done
Killing orphaned processes on port 9092: 1589790
Killing orphaned processes on port 9093: 1589597
1590351
1590352
1590353
🚀 Restarting Kafka components…
[INFO] 🚀 starting confluent-kraft-identity-sau-main-dev_coordinator.service…
[2026-01-02 06:54:18 UTC] USER=www-data EUID=0 PID=1594571 ACTION=passthru ARGS=systemctl restart confluent-kraft-identity-sau-main-dev_coordinator.service
[INFO] 🚀 starting confluent-connect-identity-sau-main-dev_coordinator.service…
[2026-01-02 06:54:19 UTC] USER=www-data EUID=0 PID=1595098 ACTION=passthru ARGS=systemctl restart confluent-connect-identity-sau-main-dev_coordinator.service
[INFO] ⏳ Waiting for Kafka broker readiness (FQDN: eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com, IP: 10.100.1.212) ...
[OK] ✅ Broker ready (attempt 1)
[OK] ✅ Port 9092 listening (Kafka Broker)
[INFO] ⏳ waiting for Kafka Connect REST port 8083 … (1/40)
[OK] ✅ Port 8083 listening (Kafka Connect REST)
[INFO] ⏳ Waiting for Connect REST at https://eventbus-identity-sau-main-dev-kafka-connect.fastorder.com:8083 …
[OK] ✅ Connect REST is up (attempt 1)
📋 Reconciling Connect internal topics…
[ok] connect-configs exists
[ok] connect-offsets exists
[ok] connect-status exists
═══════════════════════════════════════════════════════════════════
KAFKA SUMMARY
═══════════════════════════════════════════════════════════════════
Env: identity-sau-main-dev Identifier: coordinator Mode: kraft
Broker Unit : confluent-kraft-identity-sau-main-dev_coordinator.service (status: active)
Connect Unit: confluent-connect-identity-sau-main-dev_coordinator.service (status: active)
Bootstrap : eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092
Connect URL : https://eventbus-identity-sau-main-dev-kafka-connect.fastorder.com:8083
═══════════════════════════════════════════════════════════════════
[OK] ✅ All required services are up.
[0;32m[OK][0m ✅ Step 4 completed: 03-restart-kafka-related-services.sh
[0;34m[INFO][0m 📦 Step 5/9: checking services...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[2026-01-02 06:55:01 UTC] USER=www-data EUID=0 PID=1597974 ACTION=fsop ARGS=test -f /opt/kafka/config/identity-sau-main-dev/coordinator/server.properties
[2026-01-02 06:55:01 UTC] USER=www-data EUID=0 PID=1597983 ACTION=passthru ARGS=bash -c grep -E '^[[:space:]]*process\.roles=' '/opt/kafka/config/identity-sau-main-dev/coordinator/server.properties' | grep -Eq '(broker|controller)'
[INFO] Detected mode from server.properties: kraft
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Step 1: Service status
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[OK] confluent-kraft-identity-sau-main-dev_coordinator.service status: active
[WARN] confluent-kafka-zk-identity-sau-main-dev_coordinator.service present but should be stopped in KRaft
[WARN] confluent-zookeeper-identity-sau-main-dev_coordinator.service present but not required in KRaft
[OK] confluent-connect-identity-sau-main-dev_coordinator.service status: active
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Step 2: Port checks
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[OK] ✅ Port 9092 listening (Kafka Broker)
[OK] ✅ Port 8083 listening (Kafka Connect REST)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Step 3: Broker readiness
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[OK] Broker API responding (attempt 1)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Step 4: Kafka Connect REST
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[OK] Connect REST responding (attempt 1)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Kafka Services Summary
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment : identity-sau-main-dev
Identifier : coordinator
Mode : kraft
Broker Unit : confluent-kraft-identity-sau-main-dev_coordinator.service (status: active)
Connect Unit: confluent-connect-identity-sau-main-dev_coordinator.service (status: active)
Broker FQDN : eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092
Broker IP : eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092
Connect URL : https://eventbus-identity-sau-main-dev-kafka-connect.fastorder.com:8083
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[OK] ✅ All required services are reachable.
[0;32m[OK][0m ✅ Step 5 completed: 04-checking-services.sh
[0;34m[INFO][0m 📦 Step 6/9: create audit topic...
🔑 Configuring AWS credentials...
✅ Using permanent AWS credentials from /home/ab/.aws/credentials
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Creating Kafka Audit Topics
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Replication Factor: 1
[0;34m[INFO][0m Waiting for Kafka to be ready...
[0;31m[ERROR][0m Kafka not ready after 60s. Skipping audit topic creation.
[0;32m[OK][0m ✅ Step 6 completed: 05-create-audit-topic.sh
[0;34m[INFO][0m 📦 Step 7/9: setup backups...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Kafka Backup Configuration
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Environment: identity-sau-main-dev
🔑 Configuring AWS credentials...
✅ Using permanent AWS credentials from /home/ab/.aws/credentials
[0;34m[INFO][0m 1️⃣ Creating S3 bucket for Kafka backups...
make_bucket failed: s3://fastorder-kafka-backups-identity-sau-main-dev An error occurred (AccessDenied) when calling the CreateBucket operation: User: arn:aws:iam::464621692046:user/fo-dev is not authorized to perform: s3:CreateBucket on resource: "arn:aws:s3:::fastorder-kafka-backups-identity-sau-main-dev" because no identity-based policy allows the s3:CreateBucket action
An error occurred (NoSuchBucket) when calling the PutBucketVersioning operation: The specified bucket does not exist
Parameter validation failed:
Unknown parameter in LifecycleConfiguration.Rules[0]: "Id", must be one of: Expiration, ID, Prefix, Filter, Status, Transitions, NoncurrentVersionTransitions, NoncurrentVersionExpiration, AbortIncompleteMultipartUpload
[0;32m[OK][0m ✅ S3 bucket created: fastorder-kafka-backups-identity-sau-main-dev
[0;34m[INFO][0m 2️⃣ Creating local backup directory...
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1601978 ACTION=fsop ARGS=mkdir -p /var/backups/kafka/identity-sau-main-dev
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1601987 ACTION=fsop ARGS=mkdir -p /var/backups/kafka/identity-sau-main-dev/topics
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1601996 ACTION=fsop ARGS=mkdir -p /var/backups/kafka/identity-sau-main-dev/metadata
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602005 ACTION=fsop ARGS=mkdir -p /var/log/kafka/backups
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602015 ACTION=fsop ARGS=chown -R kafka:kafka /var/backups/kafka/identity-sau-main-dev
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602024 ACTION=fsop ARGS=chown -R kafka:kafka /var/log/kafka/backups
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602033 ACTION=fsop ARGS=chmod 750 /var/backups/kafka/identity-sau-main-dev
[0;32m[OK][0m ✅ Local backup directory created
[0;34m[INFO][0m 3️⃣ Creating topic backup script...
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602051 ACTION=fsop ARGS=sed -i s|__ENV_ID__|identity-sau-main-dev|g /usr/local/bin/kafka-backup-identity-sau-main-dev.sh
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602060 ACTION=fsop ARGS=sed -i s|__KAFKA_BROKER__|eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com:9092|g /usr/local/bin/kafka-backup-identity-sau-main-dev.sh
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602069 ACTION=fsop ARGS=sed -i s|__BACKUP_DIR__|/var/backups/kafka/identity-sau-main-dev|g /usr/local/bin/kafka-backup-identity-sau-main-dev.sh
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602078 ACTION=fsop ARGS=sed -i s|__S3_BUCKET__|fastorder-kafka-backups-identity-sau-main-dev|g /usr/local/bin/kafka-backup-identity-sau-main-dev.sh
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602087 ACTION=fsop ARGS=sed -i s|__S3_REGION__|me-central-1|g /usr/local/bin/kafka-backup-identity-sau-main-dev.sh
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602096 ACTION=fsop ARGS=chmod 750 /usr/local/bin/kafka-backup-identity-sau-main-dev.sh
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602105 ACTION=fsop ARGS=chown root:kafka /usr/local/bin/kafka-backup-identity-sau-main-dev.sh
[0;32m[OK][0m ✅ Backup script created: /usr/local/bin/kafka-backup-identity-sau-main-dev.sh
[0;34m[INFO][0m 4️⃣ Setting up cron jobs for automated backups...
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602123 ACTION=fsop ARGS=chmod 644 /etc/cron.d/kafka-backups-identity-sau-main-dev
[0;32m[OK][0m ✅ Cron job configured: Daily backups at 2:00 AM
[0;34m[INFO][0m 5️⃣ Creating restore documentation...
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602141 ACTION=fsop ARGS=sed -i s|__S3_BUCKET__|fastorder-kafka-backups-identity-sau-main-dev|g /var/backups/kafka/identity-sau-main-dev/RESTORE_INSTRUCTIONS.md
[2026-01-02 06:57:28 UTC] USER=www-data EUID=0 PID=1602153 ACTION=fsop ARGS=sed -i s|__S3_REGION__|me-central-1|g /var/backups/kafka/identity-sau-main-dev/RESTORE_INSTRUCTIONS.md
[2026-01-02 06:57:29 UTC] USER=www-data EUID=0 PID=1602164 ACTION=fsop ARGS=sed -i s|__KAFKA_BROKER__|eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com|g /var/backups/kafka/identity-sau-main-dev/RESTORE_INSTRUCTIONS.md
[2026-01-02 06:57:29 UTC] USER=www-data EUID=0 PID=1602173 ACTION=fsop ARGS=chmod 644 /var/backups/kafka/identity-sau-main-dev/RESTORE_INSTRUCTIONS.md
[2026-01-02 06:57:29 UTC] USER=www-data EUID=0 PID=1602182 ACTION=fsop ARGS=chown kafka:kafka /var/backups/kafka/identity-sau-main-dev/RESTORE_INSTRUCTIONS.md
[0;32m[OK][0m ✅ Restore documentation created: /var/backups/kafka/identity-sau-main-dev/RESTORE_INSTRUCTIONS.md
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m ✅ Kafka Backup Configured
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m S3 Bucket: fastorder-kafka-backups-identity-sau-main-dev
[0;34m[INFO][0m Region: me-central-1
[0;34m[INFO][0m Local backup dir: /var/backups/kafka/identity-sau-main-dev
[0;34m[INFO][0m Schedule: Daily at 2:00 AM
[0;34m[INFO][0m Script: /usr/local/bin/kafka-backup-identity-sau-main-dev.sh
[0;34m[INFO][0m Restore docs: /var/backups/kafka/identity-sau-main-dev/RESTORE_INSTRUCTIONS.md
[1;33m[WARN][0m ⚠️ Note: This backs up Kafka metadata only (topics, configs, offsets)
[1;33m[WARN][0m For full message data backup, configure Kafka Connect S3 Sink
[0;32m[OK][0m ✅ Step 7 completed: 06-setup-backups.sh
[0;34m[INFO][0m 📦 Step 8/9: monitoring setup...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] 🔍 Kafka Monitoring Integration for identity-sau-main-dev
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] 1️⃣ Checking observability cell readiness...
[INFO] Checking observability cell readiness: obs-identity-sau-main-dev
[1;32m[OK][0m Observability cell endpoints registered for identity-sau-main-dev
[1;32m[OK][0m ✓ Observability cell is ready
[INFO] 2️⃣ Setting up Kafka JMX exporter integration...
[INFO] Checking observability cell readiness: obs-identity-sau-main-dev
[1;32m[OK][0m Observability cell endpoints registered for identity-sau-main-dev
[INFO] Setting up Kafka JMX exporter for identity-sau-main-dev
[INFO] JMX Prometheus Java Agent already exists at /opt/kafka/libs/jmx_prometheus_javaagent.jar
[2026-01-02 06:57:29 UTC] USER=www-data EUID=0 PID=1602230 ACTION=passthru ARGS=mv /tmp/jmx_exporter.yml /opt/kafka/config/jmx_exporter.yml
[2026-01-02 06:57:29 UTC] USER=www-data EUID=0 PID=1602239 ACTION=passthru ARGS=chmod 644 /opt/kafka/config/jmx_exporter.yml
[1;32m[OK][0m JMX exporter configuration created at /opt/kafka/config/jmx_exporter.yml
[1;32m[OK][0m JMX exporter configuration created
[INFO] Configuring Kafka systemd services to use JMX exporter...
[2026-01-02 06:57:29 UTC] USER=www-data EUID=0 PID=1602266 ACTION=fsop ARGS=test -f /etc/systemd/system/[2026-01-02
[INFO] All Kafka services already configured with JMX exporter
[1;32m[OK][0m Kafka JMX exporter integration complete
[INFO] Metrics endpoint: http://142.93.238.16:9308/metrics
[INFO] Prometheus will automatically scrape: https://metrics-identity-sau-main-dev.fastorder.com:9090
[INFO] View dashboards at: https://dashboards-identity-sau-main-dev.fastorder.com
[1;32m[OK][0m ✓ Kafka JMX exporter integration complete
[INFO] Configuring KAFKA_OPTS environment variable for kafka user...
[2026-01-02 06:57:30 UTC] USER=www-data EUID=0 PID=1602289 ACTION=passthru ARGS=grep -q KAFKA_OPTS.*javaagent.*jmx_prometheus_javaagent /home/kafka/.bashrc
[1;32m[OK][0m ✓ KAFKA_OPTS already configured
[INFO] 2.5️⃣ Enabling JMX exporter in Kafka systemd service...
[2026-01-02 06:57:30 UTC] USER=www-data EUID=0 PID=1602314 ACTION=passthru ARGS=grep -q javaagent.*jmx_prometheus_javaagent /etc/systemd/system/confluent-kraft-identity-sau-main-dev_coordinator.service
[INFO] Updating confluent-kraft-identity-sau-main-dev_coordinator.service to enable JMX exporter...
[2026-01-02 06:57:30 UTC] USER=www-data EUID=0 PID=1602338 ACTION=passthru ARGS=sed -i s|^Environment=KAFKA_OPTS=.*|Environment=KAFKA_OPTS=-javaagent:/opt/kafka/libs/jmx_prometheus_javaagent.jar=9308:/opt/kafka/config/jmx_exporter.yml| /etc/systemd/system/confluent-kraft-identity-sau-main-dev_coordinator.service
[1;32m[OK][0m ✓ Updated confluent-kraft-identity-sau-main-dev_coordinator.service
[INFO] Reloading systemd daemon and restarting Kafka services...
[2026-01-02 06:57:30 UTC] USER=www-data EUID=0 PID=1602359 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 06:57:31 UTC] USER=www-data EUID=0 PID=1602422 ACTION=passthru ARGS=systemctl is-active --quiet confluent-kraft-identity-sau-main-dev_coordinator
[INFO] Restarting confluent-kraft-identity-sau-main-dev_coordinator...
[2026-01-02 06:57:31 UTC] USER=www-data EUID=0 PID=1602443 ACTION=passthru ARGS=systemctl restart confluent-kraft-identity-sau-main-dev_coordinator
[2026-01-02 06:57:36 UTC] USER=www-data EUID=0 PID=1602961 ACTION=passthru ARGS=systemctl is-active --quiet confluent-kraft-identity-sau-main-dev_coordinator
[1;32m[OK][0m ✓ confluent-kraft-identity-sau-main-dev_coordinator restarted successfully
[1;32m[OK][0m ✓ JMX exporter enabled in Kafka systemd services
[INFO] 2.6️⃣ Configuring Prometheus to scrape Kafka metrics...
[2026-01-02 06:57:36 UTC] USER=www-data EUID=0 PID=1602982 ACTION=passthru ARGS=grep -q job_name: 'kafka' /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
[INFO] Adding Kafka scrape target to Prometheus configuration...
[ERROR] No passwordless sudo and wrapper does not allow 'bash'. Run as root or extend wrapper.
[2026-01-02 06:57:36 UTC] USER=www-data EUID=0 PID=1603021 ACTION=passthru ARGS=sed -i /# Prometheus self-monitoring/r /tmp/prometheus_kafka_add.yml /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
[ERROR] Invalid Prometheus configuration - rolling back
[2026-01-02 06:57:36 UTC] USER=www-data EUID=0 PID=1603057 ACTION=passthru ARGS=sed -i /job_name: 'kafka'/,+6d /etc/prometheus/obs-identity-sau-main-dev/prometheus.yml
[2026-01-02 06:57:36 UTC] USER=www-data EUID=0 PID=1603078 ACTION=fsop ARGS=rm -f /tmp/prometheus_kafka_add.yml
[INFO] 3️⃣ Registering Kafka nodes to monitoring database...
[INFO] Registering Kafka Broker to monitoring dashboard...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: Kafka Broker
[INFO] Identifier: identity-sau-main-dev-broker-01
[INFO] Identifier Parent: cluster
[INFO] IP: 142.93.238.16
[INFO] Port: 9092
[INFO] FQDN: eventbus-identity-sau-main-dev-kafka-broker-01.fastorder.com
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: 1a310579-24b9-4091-8626-7335f80305c3
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[1;32m[OK][0m ✓ Kafka broker registered
[INFO] Registering Kafka Connect to monitoring dashboard...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: Kafka Connect
[INFO] Identifier: identity-sau-main-dev-connect-01
[INFO] Identifier Parent: cluster
[INFO] IP: 142.93.238.16
[INFO] Port: 8083
[INFO] FQDN: eventbus-identity-sau-main-dev-kafka-connect.fastorder.com
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: 71719f62-65ea-4a2b-a0ed-4a8d3f80403b
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[1;32m[OK][0m ✓ Kafka Connect registered
[INFO] Schema Registry not running, skipping registration
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] ✅ Kafka Monitoring Setup Complete
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] Metrics: http://localhost:9308/metrics
[INFO] Prometheus: https://metrics-identity-sau-main-dev.fastorder.com:9090
[INFO] Grafana: https://dashboards-identity-sau-main-dev.fastorder.com
[INFO] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m ✅ Step 8 completed: 10-monitoring-setup.sh
[0;34m[INFO][0m 📦 Step 9/9: update www data certs...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
📋 Post-Kafka Setup: Updating www-data Kafka certificates...
Environment: identity-sau-main-dev
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
✓ Kafka certificates found
✓ www-data user exists
[2026-01-02 06:57:39 UTC] USER=www-data EUID=0 PID=1603294 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:39 UTC] USER=www-data EUID=0 PID=1603304 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:39 UTC] USER=www-data EUID=0 PID=1603314 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 06:57:39 UTC] USER=www-data EUID=0 PID=1603323 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
✅ Symlinked client-key.pem
[2026-01-02 06:57:39 UTC] USER=www-data EUID=0 PID=1603345 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:39 UTC] USER=www-data EUID=0 PID=1603355 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:57:39 UTC] USER=www-data EUID=0 PID=1603364 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:57:39 UTC] USER=www-data EUID=0 PID=1603377 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks created for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✓ Post-Kafka setup complete
[0;32m[OK][0m ✅ Step 9 completed: 99-update-www-data-certs.sh
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;32m[OK][0m ✅ Kafka setup completed successfully!
[0;32m[OK][0m Executed all 9 steps
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Service: identity
[0;34m[INFO][0m Zone: sau
[0;34m[INFO][0m Branch: main
[0;34m[INFO][0m Env: dev
[0;34m[INFO][0m Registering Kafka nodes via API...
[0;32m[OK][0m ✔ Kafka node registration completed
[0;34m[INFO][0m Setting up Kafka observability integration...
[0;34m[INFO][0m Checking observability cell readiness: obs-identity-sau-main-dev
[0;32m[OK][0m Observability cell endpoints registered for identity-sau-main-dev
[0;34m[INFO][0m Observability cell verified for identity-sau-main-dev
[0;34m[INFO][0m Monitoring will be configured after Kafka deployment (step 10-monitoring-setup.sh)
[0;34m[INFO][0m Cleaning up temporary files...
[INFO] Starting cleanup of temporary files...
[INFO] Cleaning up SSL temp files for identity-sau-main-dev...
[INFO] Cleaning up old provisioning logs...
[SUCCESS] Removed 6 old log files
[INFO] Cleaning up old configuration backups...
[0;32m[OK][0m ✔ Cleanup completed
[0;32m✓[0m ✅ Event bus infrastructure (kafka) setup completed successfully
[0;34m[INFO][0m Using database engine from DB_ENGINE environment variable: postgresql
[0;34m[INFO][0m Cleaning up any existing locks...
[0;32m[1mStarting database engine: postgresql[0m
[1;33m═══════════════════════════════════════════════[0m
[0;34m[INFO][0m Loaded from topology.json: identity-sau-main-dev
[0;32m[2026-01-02 06:57:41][0m Loaded environment: identity-sau-main-dev
[0;32m[2026-01-02 06:57:41][0m Service: identity, Zone: sau, Branch: main, Env: dev
[0;32m[2026-01-02 06:57:41][0m VM IP: 142.93.238.16, Interface: eth0:16
[0;32m[2026-01-02 06:57:41][0m Elasticsearch Nodes: 1, PostgreSQL Workers: 1
[0;32m[2026-01-02 06:57:41][0m PostgreSQL HA Nodes: 1, Citus Enabled: yes
[0;32m✓[0m Environment initialized successfully (mode: general)
[0;34m[INFO][0m Checking observability cell readiness: obs-identity-sau-main-dev
[1;32m[OK][0m Observability cell endpoints registered for identity-sau-main-dev
[0;34m[INFO][0m Observability cell verified for identity-sau-main-dev
[0;34m[INFO][0m Monitoring will be configured after PostgreSQL deployment (step 10-monitoring-setup.sh)
[0;34m[INFO][0m Citus mode ENABLED
[0;34m[INFO][0m → Coordinator + 1 worker(s) + 1 standby node(s) per worker
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Setting up coordinator (Citus control plane)…
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] 📁 Initializing log directories...
[2026-01-02 06:57:42 UTC] USER=unknown EUID=33 PID=1603558 ACTION=fsop ARGS=mkdir -p /var/log/fastorder/metrics
[2026-01-02 06:57:42 UTC] USER=unknown EUID=33 PID=1603565 ACTION=fsop ARGS=mkdir -p /var/log/fastorder/audit
[2026-01-02 06:57:43 UTC] USER=unknown EUID=33 PID=1603572 ACTION=fsop ARGS=mkdir -p /var/log/fastorder/provisioning
[2026-01-02 06:57:43 UTC] USER=unknown EUID=33 PID=1603580 ACTION=fsop ARGS=chmod 777 /var/log/fastorder/metrics
[2026-01-02 06:57:43 UTC] USER=unknown EUID=33 PID=1603589 ACTION=fsop ARGS=chmod 777 /var/log/fastorder/audit
[2026-01-02 06:57:43 UTC] USER=unknown EUID=33 PID=1603600 ACTION=fsop ARGS=chmod 777 /var/log/fastorder/provisioning
/opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/run.sh: line 41: ok: command not found
[INFO] 🟢 Starting PostgreSQL provisioning for identity in sau-dev...
[INFO] Environment: identity-sau-main-dev
[INFO] Identifier: coordinator
[INFO] VM IP: 142.93.238.16
[DEBUG] RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3 JOB_UUID=438ff2f1-77b4-4b8d-aa09-3d1698934d76
[DEBUG] Tracking substep start: steps/01-install/steps/00-configure-network-hosts (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 00 configure network hosts...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m CONFIGURING POSTGRESQL NETWORK & DNS
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: coordinator
[0;34m[INFO][0m PostgreSQL IP: 10.100.1.203
[0;34m[INFO][0m Primary hostname: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
[0;34m[INFO][0m Adding /etc/hosts entries for coordinator...
[0;34m[INFO][0m 1. db-identity-sau-main-dev-postgresql.fastorder.com → 10.100.1.203 (primary/short)
[0;34m[INFO][0m 2. db-identity-sau-main-dev-postgresql-coordinator.fastorder.com → 10.100.1.203 (compatibility)
[0;34m[INFO][0m ➕ Adding db-identity-sau-main-dev-postgresql.fastorder.com → 10.100.1.203
[0;32m✅[0m ✅ Added: db-identity-sau-main-dev-postgresql.fastorder.com → 10.100.1.203
[0;34m[INFO][0m ➕ Adding db-identity-sau-main-dev-postgresql-coordinator.fastorder.com → 10.100.1.203
[0;32m✅[0m ✅ Added: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com → 10.100.1.203
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;32m✅[0m ✅ Network & DNS configuration complete
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Verifying /etc/hosts entries:
10.100.1.203 db-identity-sau-main-dev-postgresql.fastorder.com
10.100.1.203 db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
[DEBUG] Tracking substep start: steps/01-install/steps/01-prepare-ssl-server-postgres (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 01 prepare ssl server postgres...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
📦 PostgreSQL Server Certificate Generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau (Saudi Arabia)
Branch: main
Env: dev
Node: coordinator
Primary CN: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Alt CN: identity-sau-main-dev.fastorder.com
VM IP: 142.93.238.16
Coordinator variants:
- db-identity-sau-main-dev-postgresql-coordinator-coordinator.fastorder.com
- db-sau-main-dev-postgresql-coordinator-coordinator.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ Removing existing server certificates (preserving client certs)...
[2026-01-02 06:57:47 UTC] USER=www-data EUID=0 PID=1603804 ACTION=fsop ARGS=rm -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
✅ Ensuring directories exist: /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator and /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
🔐 Generating 4096-bit private key...
[2026-01-02 06:57:48 UTC] USER=www-data EUID=0 PID=1603822 ACTION=fsop ARGS=chmod 755 /tmp/pg-cert-gen-1603760
[2026-01-02 06:57:48 UTC] USER=www-data EUID=0 PID=1603832 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-cert-gen-1603760/ra_root.crt
[2026-01-02 06:57:48 UTC] USER=www-data EUID=0 PID=1603841 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-cert-gen-1603760/ra_root.key
[2026-01-02 06:57:48 UTC] USER=www-data EUID=0 PID=1603850 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1603760/ra_root.crt
[2026-01-02 06:57:48 UTC] USER=www-data EUID=0 PID=1603859 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1603760/ra_root.key
📝 Creating certificate signing request (CSR)...
📜 Signing certificate with internal CA...
Certificate request self-signature ok
subject=C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1603897 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1603760/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1603910 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1603760/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1603932 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
📋 Setting up CA certificate...
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1603964 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1603760/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1604001 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1604031 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1604055 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt
✅ Using CA certificate: /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt
🚚 Setting up key in private directory...
Key already in correct location (CERT_DIR == KEY_DIR)
🔒 Securing key and cert permissions...
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1604066 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1604076 ACTION=fsop ARGS=chmod 600 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1604085 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1604094 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1604103 ACTION=fsop ARGS=chown root:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:57:49 UTC] USER=www-data EUID=0 PID=1604112 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
🔍 Verifying certificate...
Certificate details:
Subject: C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
--
X509v3 Subject Alternative Name:
DNS:db-identity-sau-main-dev-postgresql-coordinator.fastorder.com, DNS:identity-sau-main-dev.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-coordinator.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-coordinator, DNS:localhost, DNS:db-identity-sau-main-dev-postgresql-coordinator-coordinator.fastorder.com, DNS:db-sau-main-dev-postgresql-coordinator-coordinator.fastorder.com, DNS:db-identity-sau-main-dev-postgresql.fastorder.com, IP Address:142.93.238.16, IP Address:127.0.0.1
X509v3 Subject Key Identifier:
⚠️ Certificate chain verification: FAILED (but certificate may still work)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ PostgreSQL Server Certificate Generated Successfully!
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Node: coordinator
Primary CN: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Certificate files installed:
📜 Server cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
🔑 Server key: /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
🏛️ CA cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt (ca.crt symlink also available)
To use these certificates in PostgreSQL:
1. Update postgresql.conf:
ssl = on
ssl_cert_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt'
ssl_key_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key'
ssl_ca_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt'
2. Restart PostgreSQL:
command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh passthru systemctl restart postgresql@identity-sau-main-dev-coordinator.service
3. Test SSL connection:
psql "host=db-identity-sau-main-dev-postgresql-coordinator.fastorder.com port=5432 user=postgres sslmode=verify-full"
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: postgres
Identifier: coordinator
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: coordinator
User (CN): postgres
Hostname: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604177 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-coordinator-postgres
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604186 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-coordinator-postgres/ra_root.crt
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604195 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-coordinator-postgres/ra_root.key
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604204 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-coordinator-postgres/ra_root.crt
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604213 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-coordinator-postgres/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = postgres
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604228 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604237 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604246 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/postgres.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604255 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt
[2026-01-02 06:57:50 UTC] USER=www-data EUID=0 PID=1604264 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604273 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604282 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/postgres.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604291 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/postgres_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604300 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604314 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604323 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604332 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604341 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604350 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604359 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604369 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604398 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604409 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604418 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604430 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604439 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604448 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:51 UTC] USER=www-data EUID=0 PID=1604457 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604466 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604475 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604484 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604493 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604503 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604515 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604531 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604540 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604550 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604561 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604570 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604579 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604588 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604608 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604619 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604629 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604640 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:52 UTC] USER=www-data EUID=0 PID=1604649 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604660 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604670 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604679 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604688 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604697 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604706 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604715 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604724 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604734 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604744 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604754 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604764 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604773 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604782 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604791 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604800 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604809 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604818 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604827 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604836 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604845 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604855 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604865 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604874 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:53 UTC] USER=www-data EUID=0 PID=1604883 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 06:57:54 UTC] USER=www-data EUID=0 PID=1604892 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 06:57:54 UTC] USER=www-data EUID=0 PID=1604901 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 06:57:54 UTC] USER=www-data EUID=0 PID=1604910 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:54 UTC] USER=www-data EUID=0 PID=1604919 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:57:54 UTC] USER=www-data EUID=0 PID=1604928 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:57:54 UTC] USER=www-data EUID=0 PID=1604937 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: postgres
Node: coordinator
FQDN: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-coordinator.fastorder.com -U postgres -d postgres
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: postgres
Identifier: coordinator
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: coordinator
User (CN): postgres
Hostname: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 06:57:55 UTC] USER=www-data EUID=0 PID=1604991 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-coordinator-postgres
[2026-01-02 06:57:55 UTC] USER=www-data EUID=0 PID=1605001 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-coordinator-postgres/ra_root.crt
[2026-01-02 06:57:55 UTC] USER=www-data EUID=0 PID=1605010 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-coordinator-postgres/ra_root.key
[2026-01-02 06:57:55 UTC] USER=www-data EUID=0 PID=1605019 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-coordinator-postgres/ra_root.crt
[2026-01-02 06:57:55 UTC] USER=www-data EUID=0 PID=1605028 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-coordinator-postgres/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = postgres
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605046 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605055 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605064 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/postgres.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605073 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605082 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605091 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605101 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/postgres.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605114 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-postgres/postgres_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605123 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605160 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605169 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605178 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605187 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605196 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605206 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605236 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605245 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:56 UTC] USER=www-data EUID=0 PID=1605254 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605265 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605275 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605284 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605302 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605311 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605320 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605331 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605341 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605351 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605360 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605370 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605380 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605389 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605398 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605407 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605416 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605425 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605434 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605443 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605453 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605463 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605472 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605481 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605490 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:57 UTC] USER=www-data EUID=0 PID=1605499 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605508 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605517 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605526 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605535 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605544 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605553 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605563 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605576 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605588 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605597 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605606 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605615 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605624 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605633 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605642 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605651 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605661 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key.pkcs1
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605671 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/postgres_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres_der.key
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605682 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:58 UTC] USER=www-data EUID=0 PID=1605695 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:59 UTC] USER=www-data EUID=0 PID=1605705 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:59 UTC] USER=www-data EUID=0 PID=1605715 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
✅ Symlinked client-cert.pem
[2026-01-02 06:57:59 UTC] USER=www-data EUID=0 PID=1605734 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 06:57:59 UTC] USER=www-data EUID=0 PID=1605743 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:57:59 UTC] USER=www-data EUID=0 PID=1605752 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:57:59 UTC] USER=www-data EUID=0 PID=1605761 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:57:59 UTC] USER=www-data EUID=0 PID=1605770 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: postgres
Node: coordinator
FQDN: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-coordinator.fastorder.com -U postgres -d postgres
[DEBUG] Tracking substep start: steps/01-install/steps/02-setup-pg-instance (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 02 setup pg instance...
[0;34m[DEADLOCK-PREVENTION][0m Deadlock prevention library loaded
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
🔑 Configuring AWS credentials...
✅ Using permanent AWS credentials from /var/www/.aws/credentials
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔑 Configuring AWS credentials...
[WARN] ~/.aws/credentials file not found
[WARN] AWS operations may require SSO login
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] Creating Postgresql Ident:db-coordinator-postgresql application environment...
[INFO] 🎯 Custom Environment Creation (Example Wrapper)
[INFO] 📁 Orchestrator Library: /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator
[INFO] 💾 State Directory: /opt/fastorder/bash/scripts/env_app_setup/state
[INFO] 🚀 Calling centralized orchestrator: fo-env create-app
[INFO] 📋 Arguments: --service identity --zone sau --branch main --env dev --domain db-identity-sau-main-dev-postgresql-coordinator --app db-coordinator-postgresql
[INFO] Creating application-specific environment configuration
[INFO] Environment ID: identity-sau-main-dev
[INFO] Application: db-coordinator-postgresql
[INFO] Base environment identity-sau-main-dev already exists
[INFO] Allocated db-coordinator-postgresql IP: 10.100.1.213
[INFO] Generated domain: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
[INFO] Configuring network interface for db-coordinator-postgresql IP: 10.100.1.213
[2026-01-02 06:58:03 UTC] USER=www-data EUID=0 PID=1606271 ACTION=passthru ARGS=ip addr add 10.100.1.213/32 dev eth0 label eth0:213
[ OK ] Configured db-coordinator-postgresql IP 10.100.1.213 on interface eth0:213
[INFO] Creating systemd service for db-coordinator-postgresql IP persistence...
[2026-01-02 06:58:03 UTC] USER=www-data EUID=0 PID=1606291 ACTION=passthru ARGS=systemctl daemon-reload
[ OK ] db-coordinator-postgresql IP will persist across reboots
[INFO] Updating topology with application-specific configuration...
[ OK ] Topology updated with application-specific configuration
[INFO] Binding db-coordinator-postgresql IP to domain: 10.100.1.213 -> db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
[WARN] Domain 'db-identity-sau-main-dev-postgresql-coordinator.fastorder.com' already exists in /etc/hosts
[INFO] Removing old entries for domain: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
[2026-01-02 06:58:04 UTC] USER=www-data EUID=0 PID=1606432 ACTION=fsop ARGS=sed -i /\sdb-identity-sau-main-dev-postgresql-coordinator.fastorder.com\(\s\|$\)/d /etc/hosts
[ OK ] Successfully bound db-identity-sau-main-dev-postgresql-coordinator.fastorder.com to 10.100.1.213
[ OK ] Domain correctly mapped
[ OK ] Application environment created successfully!
[INFO]
[INFO] Application Details:
[INFO] Environment ID: identity-sau-main-dev
[INFO] Application: db-coordinator-postgresql
[INFO] IP: 10.100.1.213
[INFO] Domain: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
[INFO]
[INFO] To use this application:
[INFO] source /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/config_management.sh
[INFO] init_environment db-coordinator-postgresql
[INFO] echo $VM_IP # Returns: 10.100.1.213
[ OK ] 🎉 Environment creation completed successfully!
[INFO] 📋 What happened:
[INFO] ✅ Called centralized orchestrator at /opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator
[INFO] ✅ All topology.json management handled centrally
[INFO] ✅ Application-specific IP and domain configured
[INFO] ✅ Network interface configured and made persistent
[INFO] ✅ Domain binding added to /etc/hosts (if not skipped)
[INFO] 🔧 To use the centralized orchestrator directly:
[INFO] # Add orchestrator to PATH
[INFO] export PATH="/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/bin:$PATH"
[INFO] # Then call directly
[INFO] fo-env create-app --service auth --zone uae --env dev --app redis
[INFO] 📚 For more orchestrator commands:
[INFO] fo-env --help
[ OK ] Created db-coordinator-postgresql environment: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com (10.100.1.213)
[INFO] PostgreSQL will listen on application-specific IP: 10.100.1.213
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: coordinator
[0;34m[INFO][0m Data dir: /var/lib/postgresql/17/identity-sau-main-dev/coordinator
[0;34m[INFO][0m Port: 5432
[0;34m[INFO][0m Hostname: db-identity-sau-main-dev-postgresql-coordinator
[2026-01-02 06:58:05 UTC] USER=www-data EUID=0 PID=1606489 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:58:05 UTC] USER=www-data EUID=0 PID=1606511 ACTION=fsop ARGS=chmod 755 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:58:05 UTC] USER=www-data EUID=0 PID=1606539 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:58:05 UTC] USER=www-data EUID=0 PID=1606560 ACTION=fsop ARGS=chown root:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[1;33m[WARN][0m Server certificate not found at /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
[0;34m[INFO][0m Generating server certificate using ssl/server.sh...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
📦 PostgreSQL Server Certificate Generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau (Saudi Arabia)
Branch: main
Env: dev
Node: coordinator
Primary CN: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Alt CN: identity-sau-main-dev.fastorder.com
VM IP: 142.93.238.16
Coordinator variants:
- db-identity-sau-main-dev-postgresql-coordinator-coordinator.fastorder.com
- db-sau-main-dev-postgresql-coordinator-coordinator.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ Removing existing server certificates (preserving client certs)...
[2026-01-02 06:58:06 UTC] USER=www-data EUID=0 PID=1606608 ACTION=fsop ARGS=rm -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
✅ Ensuring directories exist: /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator and /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:58:06 UTC] USER=www-data EUID=0 PID=1606618 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
🔐 Generating 4096-bit private key...
[2026-01-02 06:58:06 UTC] USER=www-data EUID=0 PID=1606628 ACTION=fsop ARGS=chmod 755 /tmp/pg-cert-gen-1606567
[2026-01-02 06:58:06 UTC] USER=www-data EUID=0 PID=1606646 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-cert-gen-1606567/ra_root.key
[2026-01-02 06:58:06 UTC] USER=www-data EUID=0 PID=1606656 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1606567/ra_root.crt
[2026-01-02 06:58:06 UTC] USER=www-data EUID=0 PID=1606666 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1606567/ra_root.key
📝 Creating certificate signing request (CSR)...
📜 Signing certificate with internal CA...
Certificate request self-signature ok
subject=C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
[2026-01-02 06:58:09 UTC] USER=www-data EUID=0 PID=1606741 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1606567/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
[2026-01-02 06:58:09 UTC] USER=www-data EUID=0 PID=1606750 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
📋 Setting up CA certificate...
[2026-01-02 06:58:09 UTC] USER=www-data EUID=0 PID=1606759 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1606567/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606768 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606780 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606789 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt
✅ Using CA certificate: /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt
🚚 Setting up key in private directory...
Key already in correct location (CERT_DIR == KEY_DIR)
🔒 Securing key and cert permissions...
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606800 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606809 ACTION=fsop ARGS=chmod 600 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606818 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606827 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606836 ACTION=fsop ARGS=chown root:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606845 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
🔍 Verifying certificate...
Certificate details:
Subject: C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
--
X509v3 Subject Alternative Name:
DNS:db-identity-sau-main-dev-postgresql-coordinator.fastorder.com, DNS:identity-sau-main-dev.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-coordinator.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-coordinator, DNS:localhost, DNS:db-identity-sau-main-dev-postgresql-coordinator-coordinator.fastorder.com, DNS:db-sau-main-dev-postgresql-coordinator-coordinator.fastorder.com, DNS:db-identity-sau-main-dev-postgresql.fastorder.com, IP Address:142.93.238.16, IP Address:127.0.0.1
X509v3 Subject Key Identifier:
⚠️ Certificate chain verification: FAILED (but certificate may still work)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ PostgreSQL Server Certificate Generated Successfully!
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Node: coordinator
Primary CN: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Certificate files installed:
📜 Server cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
🔑 Server key: /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
🏛️ CA cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt (ca.crt symlink also available)
To use these certificates in PostgreSQL:
1. Update postgresql.conf:
ssl = on
ssl_cert_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt'
ssl_key_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key'
ssl_ca_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt'
2. Restart PostgreSQL:
command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh passthru systemctl restart postgresql@identity-sau-main-dev-coordinator.service
3. Test SSL connection:
psql "host=db-identity-sau-main-dev-postgresql-coordinator.fastorder.com port=5432 user=postgres sslmode=verify-full"
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m ✅ Using canonical certificate path (hardened, ProtectHome=true compatible)
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606875 ACTION=fsop ARGS=test -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.crt
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606886 ACTION=fsop ARGS=test -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/server.key
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606895 ACTION=fsop ARGS=test -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt
[0;32m[OK][0m mTLS certificates OK (server cert + client certs verified) and keys secured
[0;34m[INFO][0m Preflight: stopping any conflicting Postgres services/processes on port 5432…
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606916 ACTION=passthru ARGS=systemctl stop postgresql@identity-sau-main-dev-coordinator.service
[2026-01-02 06:58:10 UTC] USER=www-data EUID=0 PID=1606937 ACTION=passthru ARGS=systemctl stop postgresql
[0;32m[OK][0m No conflicting Postgres left on port 5432
[0;32m[OK][0m Generated new postgres password for initdb
[2026-01-02 06:58:39 UTC] USER=www-data EUID=0 PID=1607577 ACTION=fsop ARGS=chown postgres:postgres /tmp/.pg_pwfile.z0aL3Y
[2026-01-02 06:58:39 UTC] USER=www-data EUID=0 PID=1607599 ACTION=fsop ARGS=chmod 600 /tmp/.pg_pwfile.z0aL3Y
[2026-01-02 06:58:39 UTC] USER=www-data EUID=0 PID=1607623 ACTION=fsop ARGS=mkdir -p /var/lib/postgresql/17/identity-sau-main-dev
[2026-01-02 06:58:39 UTC] USER=www-data EUID=0 PID=1607645 ACTION=fsop ARGS=chown postgres:postgres /var/lib/postgresql/17/identity-sau-main-dev
[2026-01-02 06:58:40 UTC] USER=www-data EUID=0 PID=1607667 ACTION=fsop ARGS=chmod 755 /var/lib/postgresql/17/identity-sau-main-dev
[0;34m[INFO][0m Initializing cluster in /var/lib/postgresql/17/identity-sau-main-dev/coordinator (SCRAM; pwfile)
[2026-01-02 06:58:40 UTC] USER=www-data EUID=0 PID=1607690 ACTION=fsop ARGS=mkdir -p /var/lib/postgresql/17/identity-sau-main-dev/coordinator
[2026-01-02 06:58:40 UTC] USER=www-data EUID=0 PID=1607711 ACTION=fsop ARGS=chown postgres:postgres /var/lib/postgresql/17/identity-sau-main-dev/coordinator
[2026-01-02 06:58:40 UTC] USER=www-data EUID=0 PID=1607732 ACTION=fsop ARGS=chmod 700 /var/lib/postgresql/17/identity-sau-main-dev/coordinator
[2026-01-02 06:58:40 UTC] USER=www-data EUID=0 PID=1607753 ACTION=fsop ARGS=mkdir -p /var/run/postgresql-identity-sau-main-dev-coordinator
[2026-01-02 06:58:40 UTC] USER=www-data EUID=0 PID=1607779 ACTION=fsop ARGS=chown postgres:postgres /var/run/postgresql-identity-sau-main-dev-coordinator
[2026-01-02 06:58:40 UTC] USER=www-data EUID=0 PID=1607800 ACTION=fsop ARGS=chmod 2775 /var/run/postgresql-identity-sau-main-dev-coordinator
[2026-01-02 06:58:40 UTC] USER=www-data EUID=0 PID=1607809 ACTION=passthru ARGS=sudo -u postgres /usr/lib/postgresql/17/bin/initdb -D /var/lib/postgresql/17/identity-sau-main-dev/coordinator --locale=en_US.UTF-8 --encoding=UTF8 --auth-local=scram-sha-256 --auth-host=scram-sha-256 --pwfile=/tmp/.pg_pwfile.z0aL3Y
The files belonging to this database system will be owned by user "postgres".
This user must also own the server process.
The database cluster will be initialized with locale "en_US.UTF-8".
The default text search configuration will be set to "english".
Data page checksums are disabled.
fixing permissions on existing directory /var/lib/postgresql/17/identity-sau-main-dev/coordinator ... ok
creating subdirectories ... ok
selecting dynamic shared memory implementation ... posix
selecting default "max_connections" ... 100
selecting default "shared_buffers" ... 128MB
selecting default time zone ... Etc/UTC
creating configuration files ... ok
running bootstrap script ... ok
performing post-bootstrap initialization ... ok
syncing data to disk ... ok
Success. You can now start the database server using:
/usr/lib/postgresql/17/bin/pg_ctl -D /var/lib/postgresql/17/identity-sau-main-dev/coordinator -l logfile start
[0;32m[OK][0m initdb complete
[2026-01-02 06:58:42 UTC] USER=www-data EUID=0 PID=1607863 ACTION=fsop ARGS=rm -f /tmp/.pg_pwfile.z0aL3Y
[0;34m[INFO][0m Writing postgresql.conf (TLS≥1.2, SCRAM, audit logs)
[0;32m[OK][0m postgresql.conf updated successfully
[0;34m[INFO][0m Writing pg_hba.conf (mTLS with client certificates + SCRAM, least-privilege)
[2026-01-02 06:58:43 UTC] USER=www-data EUID=0 PID=1607920 ACTION=fsop ARGS=cp /tmp/tmp.7ywNqEEYrV /var/lib/postgresql/17/identity-sau-main-dev/coordinator/pg_hba.conf
[2026-01-02 06:58:43 UTC] USER=www-data EUID=0 PID=1607944 ACTION=fsop ARGS=chown postgres:postgres /var/lib/postgresql/17/identity-sau-main-dev/coordinator/pg_hba.conf
[2026-01-02 06:58:43 UTC] USER=www-data EUID=0 PID=1607967 ACTION=fsop ARGS=chmod 600 /var/lib/postgresql/17/identity-sau-main-dev/coordinator/pg_hba.conf
[0;32m[OK][0m pg_hba.conf updated
[0;34m[INFO][0m Creating systemd unit: /etc/systemd/system/postgresql@identity-sau-main-dev-coordinator.service
[2026-01-02 06:58:43 UTC] USER=www-data EUID=0 PID=1607993 ACTION=fsop ARGS=mv -f /tmp/.pg_unit.9arAnl /etc/systemd/system/postgresql@identity-sau-main-dev-coordinator.service
[2026-01-02 06:58:43 UTC] USER=www-data EUID=0 PID=1608016 ACTION=fsop ARGS=chmod 0644 /etc/systemd/system/postgresql@identity-sau-main-dev-coordinator.service
[0;32m[OK][0m systemd unit written
[2026-01-02 06:58:44 UTC] USER=www-data EUID=0 PID=1608037 ACTION=fsop ARGS=mkdir -p /var/lib/pgbackrest /var/spool/pgbackrest /var/log/pgbackrest
[2026-01-02 06:58:44 UTC] USER=www-data EUID=0 PID=1608058 ACTION=fsop ARGS=chown postgres:postgres /var/lib/pgbackrest /var/spool/pgbackrest /var/log/pgbackrest
[2026-01-02 06:58:44 UTC] USER=www-data EUID=0 PID=1608079 ACTION=passthru ARGS=systemctl daemon-reload
[0;34m[INFO][0m Starting PostgreSQL instance...
[2026-01-02 06:58:46 UTC] USER=www-data EUID=0 PID=1608211 ACTION=passthru ARGS=systemctl start postgresql@identity-sau-main-dev-coordinator.service
[0;34m[INFO][0m Waiting for ACTIVE (systemd)…
[2026-01-02 06:58:46 UTC] USER=www-data EUID=0 PID=1608257 ACTION=passthru ARGS=systemctl is-active --quiet postgresql@identity-sau-main-dev-coordinator.service
[0;32m[OK][0m Service ACTIVE
[0;34m[INFO][0m Waiting for port 5432 bind…
[0;32m[OK][0m Port bound
[0;34m[INFO][0m Waiting pg_isready (socket)…
[0;32m[OK][0m Readiness via socket OK
[0;34m[INFO][0m Waiting pg_isready (TCP db-identity-sau-main-dev-postgresql-coordinator.fastorder.com:5432)…
[0;32m[OK][0m Startup sequence complete
[0;34m[INFO][0m Validating core security GUCs (via local socket)…
[0;32m[OK][0m Security GUCs verified (ssl, min TLS, SCRAM, audit logs)
[0;34m[INFO][0m Provisioning application database and Debezium role (if not exists)...
[0;34m[INFO][0m Checking if database fastorder_identity_sau_main_dev_db exists...
[0;34m[INFO][0m DB check result: exit_code=0, output='[2026-01-02 06:58:48 UTC] USER=www-data EUID=0 PID=1608441 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc -Atqc SELECT 1 FROM pg_database WHERE datname = 'fastorder_identity_sau_main_dev_db''
[0;34m[INFO][0m Creating database fastorder_identity_sau_main_dev_db...
[2026-01-02 06:58:48 UTC] USER=www-data EUID=0 PID=1608464 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c CREATE DATABASE "fastorder_identity_sau_main_dev_db" ENCODING 'UTF8' LC_COLLATE 'en_US.UTF-8' LC_CTYPE 'en_US.UTF-8' TEMPLATE template0;
CREATE DATABASE
[0;32m[OK][0m Database fastorder_identity_sau_main_dev_db created
[0;34m[INFO][0m Checking if role debezium_user exists...
[0;34m[INFO][0m Role check result: exit_code=0, output='[2026-01-02 06:58:48 UTC] USER=www-data EUID=0 PID=1608490 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc -Atqc SELECT 1 FROM pg_roles WHERE rolname = 'debezium_user''
[0;34m[INFO][0m Creating role debezium_user...
[2026-01-02 06:58:49 UTC] USER=www-data EUID=0 PID=1608560 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c CREATE ROLE debezium_user LOGIN PASSWORD 'W68PF08KhtXXf7Af1uoPB1j2';
CREATE ROLE
[0;32m[OK][0m Role debezium_user created
[2026-01-02 06:58:49 UTC] USER=www-data EUID=0 PID=1608639 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c GRANT CONNECT ON DATABASE "fastorder_identity_sau_main_dev_db" TO debezium_user;
GRANT
[0;32m[OK][0m Application DB (fastorder_identity_sau_main_dev_db) + Debezium role (debezium_user) provisioned (idempotent)
[0;34m[INFO][0m Applying connection and memory optimizations...
[0;34m[INFO][0m Current settings: max_connections=100, work_mem=4MB
[0;34m[INFO][0m Target settings (coordinator): max_connections=150, work_mem=8MB
[2026-01-02 06:58:49 UTC] USER=www-data EUID=0 PID=1608722 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c ALTER SYSTEM SET max_connections = 150;
ALTER SYSTEM
[2026-01-02 06:58:50 UTC] USER=www-data EUID=0 PID=1608747 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c ALTER SYSTEM SET work_mem = '8MB';
ALTER SYSTEM
[2026-01-02 06:58:50 UTC] USER=www-data EUID=0 PID=1608774 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc -c SELECT pg_reload_conf();
pg_reload_conf
----------------
t
(1 row)
[0;32m[OK][0m Settings applied to postgresql.auto.conf
[2026-01-02 06:58:50 UTC] USER=www-data EUID=0 PID=1608791 ACTION=passthru ARGS=sudo -u postgres test -f /var/lib/postgresql/17/identity-sau-main-dev/coordinator/standby.signal
[0;34m[INFO][0m Service recently started (4s ago) - restarting to apply max_connections...
[0;34m[INFO][0m Stopping service...
[2026-01-02 06:58:50 UTC] USER=www-data EUID=0 PID=1608814 ACTION=passthru ARGS=systemctl stop postgresql@identity-sau-main-dev-coordinator.service
[0;34m[INFO][0m Waiting for port 5432 to be released...
[0;32m[OK][0m Port 5432 released
[0;34m[INFO][0m Starting service...
[2026-01-02 06:58:54 UTC] USER=www-data EUID=0 PID=1608876 ACTION=passthru ARGS=systemctl start postgresql@identity-sau-main-dev-coordinator.service
[2026-01-02 06:58:59 UTC] USER=www-data EUID=0 PID=1608981 ACTION=passthru ARGS=systemctl is-active --quiet postgresql@identity-sau-main-dev-coordinator.service
[0;32m[OK][0m ✅ Optimization complete: max_connections=150, work_mem=8MB
[0;34m[INFO][0m Setting postgres password via centralized script... for coordinator
[0;34m[INFO][0m Temporarily disabling synchronous_commit on coordinator for password setting...
[0;32m[OK][0m Disabled synchronous_commit (was: on)
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
⚠️ ~/.aws/credentials file not found
⚠️ Using environment-based AWS authentication
[0;34m╔════════════════════════════════════════════════════════════╗[0m
[0;34m║ PostgreSQL Password Rotation via AWS Secrets Manager ║[0m
[0;34m╚════════════════════════════════════════════════════════════╝[0m
[0;34mEnvironment Configuration:[0m
[0;34m Service: identity[0m
[0;34m Zone: sau[0m
[0;34m Environment: dev[0m
[0;34m Identifier: coordinator[0m
[0;34mAWS Secret: fastorder/db/identity/sau/main/dev/postgresql/coordinator[0m
[0;34mConnection Info:[0m
[0;34m Socket Dir: /var/run/postgresql-identity-sau-main-dev-coordinator[0m
[0;34m Port: 5432[0m
[0;34mTesting AWS Secrets Manager connectivity...[0m
ℹ️ Testing AWS IAM credentials...
✅ AWS IAM credentials are valid
{
"UserId": "AIDAWYLM4MSHFSCGU7QUM",
"Account": "464621692046",
"Arn": "arn:aws:iam::464621692046:user/fo-dev"
}
[0;34mMethod 1 (PREFERRED): AWS Secrets Manager Rotation[0m
[0;34m────────────────────────────────────────────────────────────[0m
[0;34mThis method uses AWS Secrets Manager's built-in rotation:[0m
[0;34m ✓ Zero-downtime (dual-password window)[0m
[0;34m ✓ Automatic rollback on failure[0m
[0;34m ✓ CloudTrail audit log[0m
[0;34m ✓ CloudWatch metrics[0m
[0;34m ✓ No secret exposure in scripts[0m
[0;34mNon-interactive mode: Proceeding with password rotation automatically[0m
[0;34mInitial setup: Using password from initdb[0m
[0;32m✓ PostgreSQL password already set during initdb[0m
[0;34mStoring password in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/coordinator[0m
ℹ️ Setting PostgreSQL credentials in vault: fastorder/db/identity/sau/main/dev/postgresql/coordinator
ℹ️ Setting secret in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/coordinator
✅ Secret created: fastorder/db/identity/sau/main/dev/postgresql/coordinator
✅ PostgreSQL credentials set in vault: fastorder/db/identity/sau/main/dev/postgresql/coordinator
[0;32m✓ Password stored in AWS Secrets Manager[0m
[0;34mVerifying new credentials...[0m
[0;32m✓ New credentials retrieved from AWS Secrets Manager[0m
[0;34mTesting PostgreSQL connection with new credentials...[0m
[0;32m✓ PostgreSQL connection successful (socket authentication)[0m
[0;32m✓ ╔════════════════════════════════════════════════════════════╗[0m
[0;32m✓ ║ Password Rotation Complete! ║[0m
[0;32m✓ ╚════════════════════════════════════════════════════════════╝[0m
[0;34mSecret: fastorder/db/identity/sau/main/dev/postgresql/coordinator[0m
[0;34mMethod: Direct Update (stored in AWS Secrets Manager)[0m
[0;34mStatus: Completed[0m
[0;34mTo retrieve credentials:[0m
[0;34m # Using Bash library[0m
[0;34m source /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh[0m
[0;34m get_pg_credentials coordinator[0m
[0;34mAudit trail: AWS CloudTrail (for Secrets Manager operations)[0m
[0;32m✓ Done![0m
[0;34m[INFO][0m Restoring synchronous_commit on coordinator...
[0;32m[OK][0m Restored synchronous_commit to: on
[0;32m[OK][0m Password set and persisted
[0;34m[INFO][0m Updating /etc/hosts with PostgreSQL hostname mappings...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m CONFIGURING POSTGRESQL NETWORK & DNS
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: coordinator
[0;34m[INFO][0m PostgreSQL IP: 10.100.1.213
[0;34m[INFO][0m Primary hostname: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
[0;34m[INFO][0m Adding /etc/hosts entries for coordinator...
[0;34m[INFO][0m 1. db-identity-sau-main-dev-postgresql.fastorder.com → 10.100.1.213 (primary/short)
[0;34m[INFO][0m 2. db-identity-sau-main-dev-postgresql-coordinator.fastorder.com → 10.100.1.213 (compatibility)
[0;34m[INFO][0m 🔄 Updating db-identity-sau-main-dev-postgresql.fastorder.com → 10.100.1.213
[0;32m✅[0m ✅ Updated: db-identity-sau-main-dev-postgresql.fastorder.com → 10.100.1.213
[0;34m[INFO][0m ✅ db-identity-sau-main-dev-postgresql-coordinator.fastorder.com already exists with correct IP
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;32m✅[0m ✅ Network & DNS configuration complete
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Verifying /etc/hosts entries:
10.100.1.213 db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
10.100.1.213 db-identity-sau-main-dev-postgresql.fastorder.com
[0;32m[OK][0m PostgreSQL 'identity-sau-main-dev' is up with TLS/SCRAM/logging.
Superuser TCP mode: cert
Connect (mTLS):
psql "sslmode=verify-full sslrootcert=/etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt \
sslcert=/home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.crt \
sslkey=/home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/postgres.key \
host=db-identity-sau-main-dev-postgresql-coordinator port=5432 dbname=postgres user=postgres"
File been compeleted perfectly: 02-setup-pg-instance
[0;34m[INFO][0m Registering PostgreSQL node to observability API...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: PostgreSQL
[INFO] Identifier: identity-sau-main-dev-postgresql-coordinator
[INFO] Identifier Parent: coordinator
[INFO] IP: 10.100.1.213
[INFO] Port: 5432
[INFO] FQDN: db-identity-sau-main-dev-postgresql-coordinator
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: ce097707-5ce5-40c8-a941-01512555cab8
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[0;32m[OK][0m PostgreSQL node registered to observability API
[DEBUG] Tracking substep start: steps/01-install/steps/03-role (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 03 role...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[2026-01-02 06:59:17 UTC] USER=www-data EUID=0 PID=1609701 ACTION=fsop ARGS=test -f /var/lib/postgresql/17/identity-sau-main-dev/coordinator/standby.signal
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: debezium_user
Identifier: coordinator
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: coordinator
User (CN): debezium_user
Hostname: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 06:59:43 UTC] USER=www-data EUID=0 PID=1610211 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-coordinator-debezium_user
[2026-01-02 06:59:43 UTC] USER=www-data EUID=0 PID=1610220 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-coordinator-debezium_user/ra_root.crt
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610238 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-coordinator-debezium_user/ra_root.crt
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610247 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-coordinator-debezium_user/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = debezium_user
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610267 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610276 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610285 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-debezium_user/debezium_user.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610294 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-debezium_user/debezium_user.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.crt
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610303 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-debezium_user/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610312 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610321 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-debezium_user/debezium_user.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610330 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-debezium_user/debezium_user_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user_der.key
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610339 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610348 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610357 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610367 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610377 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610386 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user_der.key
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610395 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:59:44 UTC] USER=www-data EUID=0 PID=1610404 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610430 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610439 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610448 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610457 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610470 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610481 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610490 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610499 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610508 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610517 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610526 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user_der.key
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610536 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610549 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610561 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610570 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610579 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610588 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610597 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key
[2026-01-02 06:59:45 UTC] USER=www-data EUID=0 PID=1610606 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610615 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610626 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610635 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610655 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610665 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610674 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610683 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610692 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610702 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610711 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610722 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610731 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610740 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610751 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610760 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user_der.key
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610770 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610786 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610796 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610805 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610814 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610823 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 06:59:46 UTC] USER=www-data EUID=0 PID=1610832 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610841 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610850 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610859 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610868 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key.pkcs1
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610877 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/debezium_user_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user_der.key
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610887 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610902 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610912 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610922 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610932 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610941 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610950 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610959 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610969 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 06:59:47 UTC] USER=www-data EUID=0 PID=1610979 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: debezium_user
Node: coordinator
FQDN: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/debezium_user.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-coordinator.fastorder.com -U debezium_user -d postgres
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
📦 Start executing 03-create-role.sh
📦 Setting password for user: fastorder_admin_gd
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
⚠️ ~/.aws/credentials file not found
⚠️ Using environment-based AWS authentication
[0;34m╔════════════════════════════════════════════════════════════╗[0m
[0;34m║ PostgreSQL Password Rotation via AWS Secrets Manager ║[0m
[0;34m╚════════════════════════════════════════════════════════════╝[0m
[0;34mEnvironment Configuration:[0m
[0;34m Service: identity[0m
[0;34m Zone: sau[0m
[0;34m Environment: dev[0m
[0;34m Identifier: coordinator[0m
[0;34mAWS Secret: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd[0m
[0;34mConnection Info:[0m
[0;34m Socket Dir: /var/run/postgresql-identity-sau-main-dev-coordinator[0m
[0;34m Port: 5432[0m
[0;34mTesting AWS Secrets Manager connectivity...[0m
ℹ️ Testing AWS IAM credentials...
✅ AWS IAM credentials are valid
{
"UserId": "AIDAWYLM4MSHFSCGU7QUM",
"Account": "464621692046",
"Arn": "arn:aws:iam::464621692046:user/fo-dev"
}
[0;34mMethod 1 (PREFERRED): AWS Secrets Manager Rotation[0m
[0;34m────────────────────────────────────────────────────────────[0m
[0;34mThis method uses AWS Secrets Manager's built-in rotation:[0m
[0;34m ✓ Zero-downtime (dual-password window)[0m
[0;34m ✓ Automatic rollback on failure[0m
[0;34m ✓ CloudTrail audit log[0m
[0;34m ✓ CloudWatch metrics[0m
[0;34m ✓ No secret exposure in scripts[0m
[0;34mNon-interactive mode: Proceeding with password rotation automatically[0m
[0;34mGenerating new secure password...[0m
[0;34mUser fastorder_admin_gd does not exist yet - skipping ALTER, will be created by calling script[0m
[0;32m✓ Password generated for new user: fastorder_admin_gd[0m
[0;34mStoring password in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd[0m
ℹ️ Setting PostgreSQL credentials in vault: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd
ℹ️ Setting secret in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd
✅ Secret created: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd
✅ PostgreSQL credentials set in vault: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd
[0;32m✓ Password stored in AWS Secrets Manager[0m
[0;34mVerifying new credentials...[0m
[0;32m✓ New credentials retrieved from AWS Secrets Manager[0m
[0;34mTesting PostgreSQL connection with new credentials...[0m
[0;32m✓ PostgreSQL connection successful (socket authentication)[0m
[0;32m✓ ╔════════════════════════════════════════════════════════════╗[0m
[0;32m✓ ║ Password Rotation Complete! ║[0m
[0;32m✓ ╚════════════════════════════════════════════════════════════╝[0m
[0;34mSecret: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd[0m
[0;34mMethod: Direct Update (stored in AWS Secrets Manager)[0m
[0;34mStatus: Completed[0m
[0;34mTo retrieve credentials:[0m
[0;34m # Using Bash library[0m
[0;34m source /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh[0m
[0;34m get_pg_credentials coordinator[0m
[0;34mAudit trail: AWS CloudTrail (for Secrets Manager operations)[0m
[0;32m✓ Done![0m
🔍 Retrieving password from vault with identifier: coordinator/fastorder_admin_gd
✓ Retrieved password from centralized secrets vault
🌐 Using PostgreSQL host: db-identity-sau-main-dev-postgresql.fastorder.com
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: fastorder_admin_gd
Identifier: coordinator
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: coordinator
User (CN): fastorder_admin_gd
Hostname: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:00:32 UTC] USER=www-data EUID=0 PID=1617999 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-coordinator-fastorder_admin_gd
[2026-01-02 07:00:32 UTC] USER=www-data EUID=0 PID=1618061 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-coordinator-fastorder_admin_gd/ra_root.crt
[2026-01-02 07:00:32 UTC] USER=www-data EUID=0 PID=1618221 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-coordinator-fastorder_admin_gd/ra_root.key
[2026-01-02 07:00:33 UTC] USER=www-data EUID=0 PID=1618398 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-coordinator-fastorder_admin_gd/ra_root.crt
[2026-01-02 07:00:33 UTC] USER=www-data EUID=0 PID=1618499 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-coordinator-fastorder_admin_gd/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = fastorder_admin_gd
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619293 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619316 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-fastorder_admin_gd/fastorder_admin_gd.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619325 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-fastorder_admin_gd/fastorder_admin_gd.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619335 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-fastorder_admin_gd/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619347 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619357 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-fastorder_admin_gd/fastorder_admin_gd.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619367 ACTION=fsop ARGS=cp -f /tmp/pg-client-coordinator-fastorder_admin_gd/fastorder_admin_gd_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd_der.key
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619376 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619386 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619397 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd_der.key
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619406 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619415 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619424 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key
[2026-01-02 07:00:34 UTC] USER=www-data EUID=0 PID=1619445 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd_der.key
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619454 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619464 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619493 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619503 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619513 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619522 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619532 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619541 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619551 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619561 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619572 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619581 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619591 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd_der.key
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619601 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619612 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619623 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:00:35 UTC] USER=www-data EUID=0 PID=1619633 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619642 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619655 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619664 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619673 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619683 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619693 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619703 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619713 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd_der.key
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619725 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619737 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619756 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:00:36 UTC] USER=www-data EUID=0 PID=1619766 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619778 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619787 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619797 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619807 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619818 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619827 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619836 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd_der.key
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619851 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619870 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619880 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619890 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619901 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619911 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619930 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619940 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/ca.crt
[2026-01-02 07:00:37 UTC] USER=www-data EUID=0 PID=1619951 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key.pkcs1
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator → /etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:00:38 UTC] USER=www-data EUID=0 PID=1620013 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:00:38 UTC] USER=www-data EUID=0 PID=1620059 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:00:38 UTC] USER=www-data EUID=0 PID=1620089 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:00:38 UTC] USER=www-data EUID=0 PID=1620108 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:00:38 UTC] USER=www-data EUID=0 PID=1620117 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:00:38 UTC] USER=www-data EUID=0 PID=1620127 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:00:38 UTC] USER=www-data EUID=0 PID=1620139 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:00:38 UTC] USER=www-data EUID=0 PID=1620149 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:00:38 UTC] USER=www-data EUID=0 PID=1620158 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: fastorder_admin_gd
Node: coordinator
FQDN: db-identity-sau-main-dev-postgresql-coordinator.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/fastorder_admin_gd.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-coordinator.fastorder.com -U fastorder_admin_gd -d postgres
🧱 Connecting via Unix socket to create role and database...
Socket: /var/run/postgresql-identity-sau-main-dev-coordinator:5432
📦 Creating role fastorder_admin_gd...
✅ Role fastorder_admin_gd created
ℹ️ Database fastorder_identity_sau_main_dev_db already exists, skipping creation
[2026-01-02 07:00:39 UTC] USER=www-data EUID=0 PID=1620224 ACTION=passthru ARGS=sudo -u postgres psql -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc
GRANT
✅ Role and DB created via SSL
🔐 Adding user to pg_hba.conf for SSL access...
ℹ️ Using pg_hba.conf: /var/lib/postgresql/17/identity-sau-main-dev/coordinator/pg_hba.conf
✅ Added fastorder_admin_gd to pg_hba.conf
🔄 Reloading PostgreSQL configuration...
[2026-01-02 07:00:39 UTC] USER=www-data EUID=0 PID=1620261 ACTION=passthru ARGS=systemctl reload postgresql@identity-sau-main-dev-coordinator.service
✅ PostgreSQL configuration reloaded
🧪 Testing connection for user: fastorder_admin_gd
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[0;34m=== Pre-flight Checks ===[0m
ℹ️ Testing AWS IAM credentials...
✅ AWS IAM credentials are valid
{
"UserId": "AIDAWYLM4MSHFSCGU7QUM",
"Account": "464621692046",
"Arn": "arn:aws:iam::464621692046:user/fo-dev"
}
[0;32m✓[0m AWS Secrets Manager accessible
[0;34m=== Retrieving Credentials from AWS ===[0m
ℹ️ Retrieving PostgreSQL credentials for: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd
ℹ️ Fetching secret: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd
/opt/fastorder/bash/infra_core/cache.sh: line 145: /var/cache/secrets/fastorder_db_identity_sau_main_dev_postgresql_coordinator_fastorder_admin_gd.cache.tmp.1620294: Permission denied
✅ Retrieved from secrets manager: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd
✅ PostgreSQL credentials loaded for coordinator/fastorder_admin_gd: fastorder_admin_gd@db-identity-sau-main-dev-postgresql.fastorder.com:5432/fastorder_identity_sau_main_dev_db
[0;32m✓[0m Credentials retrieved: fastorder_admin_gd@db-identity-sau-main-dev-postgresql.fastorder.com:5432/fastorder_identity_sau_main_dev_db
[0;34m╔════════════════════════════════════════════╗[0m
[0;34m║ PostgreSQL Test Suite (AWS Secrets MGR) ║[0m
[0;34m╚════════════════════════════════════════════╝[0m
[0;34m=== PostgreSQL Authentication Test ===[0m
[0;31m✗[0m PostgreSQL authentication failed
---- Error Details ----
psql: error: connection to server at "db-identity-sau-main-dev-postgresql.fastorder.com" (10.100.1.213), port 5432 failed: root certificate file "/etc/fastorder/postgresql/certs/identity-sau-main-dev/coordinator/fastorder_admin_gd/root.crt" does not exist
Either provide the file, use the system's trusted roots with sslrootcert=system, or change sslmode to disable server certificate verification.
----------------------
❌ User authentication test failed
📋 Password stored securely in AWS Secrets Manager
📋 Secret path: fastorder/db/identity/sau/main/dev/postgresql/coordinator/fastorder_admin_gd
📦 End executing 03-create-role.sh
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[2026-01-02 07:00:48 UTC] USER=www-data EUID=0 PID=1620519 ACTION=fsop ARGS=test -f /var/lib/postgresql/17/identity-sau-main-dev/coordinator/standby.signal
── fast setup ─────────────────────────────────────────────
NAME : identity-sau-main-dev
IDENTIFIER : coordinator
PG HOST : db-identity-sau-main-dev-postgresql.fastorder.com:5432
ROLE : debezium_user
DB : fastorder_identity_sau_main_dev_db
SCHEMA : auth
AUTH MODE : scram (scram=password over TLS | cert=mTLS)
SUBNET ALLOW: 10.201.0.0/16
CONNECT /32 : 142.93.238.16
SSL DIR : /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator
DNS → 10.100.1.213
CA : /home/www-data/ssl/.postgresql/identity-sau-main-dev/coordinator/root.crt
🔐 Setting password for user: debezium_user
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
⚠️ ~/.aws/credentials file not found
⚠️ Using environment-based AWS authentication
[0;34m╔════════════════════════════════════════════════════════════╗[0m
[0;34m║ PostgreSQL Password Rotation via AWS Secrets Manager ║[0m
[0;34m╚════════════════════════════════════════════════════════════╝[0m
[0;34mEnvironment Configuration:[0m
[0;34m Service: identity[0m
[0;34m Zone: sau[0m
[0;34m Environment: dev[0m
[0;34m Identifier: coordinator[0m
[0;34mAWS Secret: fastorder/db/identity/sau/main/dev/postgresql/coordinator/debezium_user[0m
[0;34mConnection Info:[0m
[0;34m Socket Dir: /var/run/postgresql-identity-sau-main-dev-coordinator[0m
[0;34m Port: 5432[0m
[0;34mTesting AWS Secrets Manager connectivity...[0m
ℹ️ Testing AWS IAM credentials...
✅ AWS IAM credentials are valid
{
"UserId": "AIDAWYLM4MSHFSCGU7QUM",
"Account": "464621692046",
"Arn": "arn:aws:iam::464621692046:user/fo-dev"
}
[0;34mMethod 1 (PREFERRED): AWS Secrets Manager Rotation[0m
[0;34m────────────────────────────────────────────────────────────[0m
[0;34mThis method uses AWS Secrets Manager's built-in rotation:[0m
[0;34m ✓ Zero-downtime (dual-password window)[0m
[0;34m ✓ Automatic rollback on failure[0m
[0;34m ✓ CloudTrail audit log[0m
[0;34m ✓ CloudWatch metrics[0m
[0;34m ✓ No secret exposure in scripts[0m
[0;34mNon-interactive mode: Proceeding with password rotation automatically[0m
[0;34mGenerating new secure password...[0m
[0;34mUser debezium_user does not exist yet - skipping ALTER, will be created by calling script[0m
[0;32m✓ Password generated for new user: debezium_user[0m
[0;34mStoring password in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/coordinator/debezium_user[0m
ℹ️ Setting PostgreSQL credentials in vault: fastorder/db/identity/sau/main/dev/postgresql/coordinator/debezium_user
ℹ️ Setting secret in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/coordinator/debezium_user
✅ Secret created: fastorder/db/identity/sau/main/dev/postgresql/coordinator/debezium_user
✅ PostgreSQL credentials set in vault: fastorder/db/identity/sau/main/dev/postgresql/coordinator/debezium_user
[0;32m✓ Password stored in AWS Secrets Manager[0m
[0;34mVerifying new credentials...[0m
[0;32m✓ New credentials retrieved from AWS Secrets Manager[0m
[0;34mTesting PostgreSQL connection with new credentials...[0m
[0;32m✓ PostgreSQL connection successful (socket authentication)[0m
[0;32m✓ ╔════════════════════════════════════════════════════════════╗[0m
[0;32m✓ ║ Password Rotation Complete! ║[0m
[0;32m✓ ╚════════════════════════════════════════════════════════════╝[0m
[0;34mSecret: fastorder/db/identity/sau/main/dev/postgresql/coordinator/debezium_user[0m
[0;34mMethod: Direct Update (stored in AWS Secrets Manager)[0m
[0;34mStatus: Completed[0m
[0;34mTo retrieve credentials:[0m
[0;34m # Using Bash library[0m
[0;34m source /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh[0m
[0;34m get_pg_credentials coordinator[0m
[0;34mAudit trail: AWS CloudTrail (for Secrets Manager operations)[0m
[0;32m✓ Done![0m
🔍 Retrieving password from vault with identifier: coordinator/debezium_user
✓ Retrieved password from secrets vault
password : (stored in AWS Secrets Manager)
🔍 TLS chain check...
🔧 Ensuring role and grants…
ℹ️ Role debezium_user exists, updating
[2026-01-02 07:01:05 UTC] USER=www-data EUID=0 PID=1621079 ACTION=passthru ARGS=sudo -u postgres psql -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d postgres --no-psqlrc
ALTER ROLE
ℹ️ Database fastorder_identity_sau_main_dev_db already exists
[2026-01-02 07:01:05 UTC] USER=www-data EUID=0 PID=1621113 ACTION=passthru ARGS=sudo -u postgres psql -h /var/run/postgresql-identity-sau-main-dev-coordinator -p 5432 -d fastorder_identity_sau_main_dev_db --no-psqlrc
CREATE SCHEMA
GRANT
GRANT
GRANT
GRANT
ALTER DEFAULT PRIVILEGES
✅ Role/DB/grants ensured.
⚠️ Could not find pg_hba.conf (skipping HBA edits): /var/lib/postgresql/17/identity-sau-main-dev/coordinator/pg_hba.conf
🧪 Testing ROLE connection (scram)...
✅ SCRAM+TLS probe OK
🎉 Done.
[DEBUG] Tracking substep start: steps/01-install/steps/05-setup-service (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 05 setup service...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
ℹ️ Service-specific setup (identity) is handled by parent script
✅ Step 5 completed (service setup delegated to 01-install/run.sh)
🔍 DEBUG_CHECKPOINT_01: Starting service-specific steps discovery
🔍 DEBUG_CHECKPOINT_02: Searching for service folders in: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps
🔍 DEBUG_CHECKPOINT_03: Found directory: destroy
🔍 DEBUG_CHECKPOINT_03: Found directory: iam
🔍 DEBUG_CHECKPOINT_04: Found run.sh in: iam
🔍 DEBUG_CHECKPOINT_03: Found directory: identity
🔍 DEBUG_CHECKPOINT_04: Found run.sh in: identity
🔍 DEBUG_CHECKPOINT_03: Found directory: lib
🔍 DEBUG_CHECKPOINT_03: Found directory: passwords
🔍 DEBUG_CHECKPOINT_03: Found directory: role
🔍 DEBUG_CHECKPOINT_03: Found directory: ssl
🔍 DEBUG_CHECKPOINT_05: Service folders found: iam identity
[INFO] 📚 Detected service folders: iam identity
🔍 DEBUG_CHECKPOINT_06: Preparing to run service: iam at /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/iam/run.sh
[DEBUG] Tracking substep start: steps/01-install/steps/iam (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 🔸 Service: iam
🔍 DEBUG_CHECKPOINT_07: About to execute /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/iam/run.sh with IDENTIFIER=coordinator IDENTIFIER_PARENT=coordinator
🔍 DEBUG_CHECKPOINT_08: Running iam in AUTO mode
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[0;35m╔════════════════════════════════════════════════════════════════════════════╗[0m
[0;35m║ IAM Database Schema Initialization ║[0m
[0;35m╚════════════════════════════════════════════════════════════════════════════╝[0m
[0;34m[INFO][0m 🟢 Starting IAM schema provisioning...
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: coordinator
[0;34m[INFO][0m VM IP: 142.93.238.16
[0;34m[INFO][0m 📚 Discovered tables: core/01-tenant core/02-realm core/03-identity core/04-device core/05-identity_account core/06-identity_mfa core/07-external_idp_link policy/01-client policy/02-resource policy/03-scope policy/04-permission policy/05-role policy/06-role_permission policy/07-identity_role policy/08-policy_rule policy/09-api_key audit/01-auth_event audit/02-admin_action audit/03-risk_decision audit/04-consent_event
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m Schema: core[0m
[0;34m Core Identity Directory (tenants, realms, identities, devices, MFA)[0m
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m[INFO][0m 🔸 Table [1/20]: core/01-tenant
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.tenant Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Identifier: coordinator
Database: fastorder_identity_sau_main_dev_db
Host: db-identity-sau-main-dev-postgresql.fastorder.com:5432
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔐 Connecting to PostgreSQL over SSL (verify-full + mTLS)...
🗄️ Checking database: fastorder_identity_sau_main_dev_db
ℹ️ Database fastorder_identity_sau_main_dev_db already exists
✅ Connected to database: fastorder_identity_sau_main_dev_db
🔧 Installing extensions...
CREATE EXTENSION
CREATE EXTENSION
CREATE EXTENSION
CREATE EXTENSION
🔧 Installing Citus extension on coordinator...
CREATE EXTENSION
✅ Citus extension installed
✅ Extensions installed
🔧 Creating utils schema...
CREATE SCHEMA
✅ Utils schema created
🔧 Installing UUIDv7 function...
✅ UUIDv7 function installed
🔧 Creating core schema...
CREATE SCHEMA
✅ Schema core created
🔧 Creating ENUM types...
DO
✅ ENUM types created
🔧 Creating core.tenant table...
CREATE TABLE
COMMENT
COMMENT
COMMENT
✅ core.tenant created
🔧 Setting up Citus distribution for core.tenant...
Creating reference table: core.tenant
create_reference_table
------------------------
(1 row)
✅ Citus distribution configured
🔧 Creating update trigger...
CREATE FUNCTION
ERROR: triggers are not supported on reference tables
ERROR: triggers are not supported on reference tables
✅ Update trigger created
✅ core.tenant initialization complete
[0;32m[OK][0m Table core/01-tenant initialized
[0;34m[INFO][0m 🔸 Table [2/20]: core/02-realm
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.realm Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.realm table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$core.realm$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ core.realm created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
NOTICE: trigger "tr_realm_updated" for relation "core.realm" does not exist, skipping
DROP TRIGGER
CREATE TRIGGER
✅ core.realm initialization complete
[0;32m[OK][0m Table core/02-realm initialized
[0;34m[INFO][0m 🔸 Table [3/20]: core/03-identity
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.identity Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.identity table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$core.identity$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.identity created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
NOTICE: trigger "tr_identity_updated" for relation "core.identity" does not exist, skipping
DROP TRIGGER
CREATE TRIGGER
✅ core.identity initialization complete
[0;32m[OK][0m Table core/03-identity initialized
[0;34m[INFO][0m 🔸 Table [4/20]: core/04-device
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.device Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.device table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$core.device$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.device created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ core.device initialization complete
[0;32m[OK][0m Table core/04-device initialized
[0;34m[INFO][0m 🔸 Table [5/20]: core/05-identity_account
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.identity_account Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.identity_account table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$core.identity_account$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.identity_account created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
NOTICE: trigger "tr_identity_account_updated" for relation "core.identity_account" does not exist, skipping
DROP TRIGGER
CREATE TRIGGER
✅ core.identity_account initialization complete
[0;32m[OK][0m Table core/05-identity_account initialized
[0;34m[INFO][0m 🔸 Table [6/20]: core/06-identity_mfa
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.identity_mfa Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.identity_mfa table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$core.identity_mfa$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.identity_mfa created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ core.identity_mfa initialization complete
[0;32m[OK][0m Table core/06-identity_mfa initialized
[0;34m[INFO][0m 🔸 Table [7/20]: core/07-external_idp_link
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.external_idp_link Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.external_idp_link table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$core.external_idp_link$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.external_idp_link created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ core.external_idp_link initialization complete
[0;32m[OK][0m Table core/07-external_idp_link initialized
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m Schema: policy[0m
[0;34m RBAC/ABAC Authorization (clients, roles, permissions, policies)[0m
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m[INFO][0m 🔸 Table [8/20]: policy/01-client
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.client Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy schema...
CREATE SCHEMA
✅ Schema policy created
🔧 Creating ENUM types...
DO
✅ ENUM types created
🔧 Creating policy.client table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$policy.client$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ policy.client created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
CREATE FUNCTION
NOTICE: trigger "tr_client_updated" for relation "policy.client" does not exist, skipping
DROP TRIGGER
CREATE TRIGGER
✅ policy.client initialization complete
[0;32m[OK][0m Table policy/01-client initialized
[0;34m[INFO][0m 🔸 Table [9/20]: policy/02-resource
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.resource Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.resource table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$policy.resource$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ policy.resource created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.resource initialization complete
[0;32m[OK][0m Table policy/02-resource initialized
[0;34m[INFO][0m 🔸 Table [10/20]: policy/03-scope
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.scope Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.scope table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$policy.scope$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ policy.scope created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
NOTICE: trigger "tr_scope_updated" for relation "policy.scope" does not exist, skipping
DROP TRIGGER
CREATE TRIGGER
✅ policy.scope initialization complete
[0;32m[OK][0m Table policy/03-scope initialized
[0;34m[INFO][0m 🔸 Table [11/20]: policy/04-permission
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.permission Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.permission table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$policy.permission$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ policy.permission created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
NOTICE: trigger "tr_permission_updated" for relation "policy.permission" does not exist, skipping
DROP TRIGGER
CREATE TRIGGER
✅ policy.permission initialization complete
[0;32m[OK][0m Table policy/04-permission initialized
[0;34m[INFO][0m 🔸 Table [12/20]: policy/05-role
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.role Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.role table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$policy.role$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ policy.role created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
NOTICE: trigger "tr_role_updated" for relation "policy.role" does not exist, skipping
DROP TRIGGER
CREATE TRIGGER
✅ policy.role initialization complete
[0;32m[OK][0m Table policy/05-role initialized
[0;34m[INFO][0m 🔸 Table [13/20]: policy/06-role_permission
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.role_permission Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.role_permission table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$policy.role_permission$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
COMMENT
✅ policy.role_permission created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.role_permission initialization complete
[0;32m[OK][0m Table policy/06-role_permission initialized
[0;34m[INFO][0m 🔸 Table [14/20]: policy/07-identity_role
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.identity_role Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.identity_role table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$policy.identity_role$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ policy.identity_role created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.identity_role initialization complete
[0;32m[OK][0m Table policy/07-identity_role initialized
[0;34m[INFO][0m 🔸 Table [15/20]: policy/08-policy_rule
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.policy_rule Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.policy_rule table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$policy.policy_rule$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ policy.policy_rule created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
NOTICE: trigger "tr_policy_rule_updated" for relation "policy.policy_rule" does not exist, skipping
DROP TRIGGER
CREATE TRIGGER
✅ policy.policy_rule initialization complete
[0;32m[OK][0m Table policy/08-policy_rule initialized
[0;34m[INFO][0m 🔸 Table [16/20]: policy/09-api_key
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.api_key Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.api_key table...
NOTICE: local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables
HINT: Executing citus_add_local_table_to_metadata($$policy.api_key$$) prevents this for the given relation, and all of the connected relations
CREATE TABLE
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ policy.api_key created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.api_key initialization complete
[0;32m[OK][0m Table policy/09-api_key initialized
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m Schema: audit[0m
[0;34m Audit & Risk Logging (auth events, admin actions, risk decisions)[0m
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m[INFO][0m 🔸 Table [17/20]: audit/01-auth_event
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.auth_event Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit schema...
CREATE SCHEMA
✅ Schema audit created
🔧 Creating ENUM types...
DO
✅ ENUM types created
🔧 Creating audit.auth_event table...
CREATE TABLE
DO
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ audit.auth_event created (partitioned)
✅ audit.auth_event initialization complete
[0;32m[OK][0m Table audit/01-auth_event initialized
[0;34m[INFO][0m 🔸 Table [18/20]: audit/02-admin_action
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.admin_action Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit.admin_action table...
CREATE TABLE
DO
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ audit.admin_action created (partitioned)
✅ audit.admin_action initialization complete
[0;32m[OK][0m Table audit/02-admin_action initialized
[0;34m[INFO][0m 🔸 Table [19/20]: audit/03-risk_decision
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.risk_decision Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit.risk_decision table...
CREATE TABLE
DO
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ audit.risk_decision created (partitioned)
✅ audit.risk_decision initialization complete
[0;32m[OK][0m Table audit/03-risk_decision initialized
[0;34m[INFO][0m 🔸 Table [20/20]: audit/04-consent_event
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.consent_event Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit.consent_event table...
CREATE TABLE
DO
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
CREATE INDEX
COMMENT
COMMENT
✅ audit.consent_event created (partitioned)
🔧 Creating partition management functions...
CREATE FUNCTION
NOTICE: relation "audit.auth_event_2026_01" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_01
NOTICE: relation "audit.auth_event_2026_02" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_02
NOTICE: Created partition: audit.auth_event_2026_03
NOTICE: Created partition: audit.auth_event_2026_04
NOTICE: relation "audit.admin_action_2026_01" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_01
NOTICE: relation "audit.admin_action_2026_02" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_02
NOTICE: Created partition: audit.admin_action_2026_03
NOTICE: Created partition: audit.admin_action_2026_04
NOTICE: relation "audit.risk_decision_2026_01" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_01
NOTICE: relation "audit.risk_decision_2026_02" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_02
NOTICE: Created partition: audit.risk_decision_2026_03
NOTICE: Created partition: audit.risk_decision_2026_04
NOTICE: relation "audit.consent_event_2026_01" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_01
NOTICE: relation "audit.consent_event_2026_02" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_02
NOTICE: Created partition: audit.consent_event_2026_03
NOTICE: Created partition: audit.consent_event_2026_04
create_monthly_partitions
---------------------------
(1 row)
CREATE VIEW
CREATE FUNCTION
COMMENT
COMMENT
✅ Partition management functions created
✅ audit.consent_event initialization complete
[0;32m[OK][0m Table audit/04-consent_event initialized
[0;35m════════════════════════════════════════════════════════════════════════════[0m
[0;32m[OK][0m ✅ IAM Schema Initialization Complete!
[0;32m[OK][0m All 20 tables initialized successfully
[0;34mSchemas created:[0m
• core - Identity directory (tenant, realm, identity, devices, MFA)
• policy - Authorization (clients, roles, permissions, policies, API keys)
• audit - Logging (auth events, admin actions, risk decisions, consent)
[0;34mDesign highlights:[0m
• Citus-ready with tenant_id distribution key
• NIST 800-63 identity compliance
• PCI DSS 4.0 audit logging
• GDPR consent tracking
• Keycloak integration via ID references
[0;35m════════════════════════════════════════════════════════════════════════════[0m
🔍 DEBUG_CHECKPOINT_06: Preparing to run service: identity at /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/run.sh
[DEBUG] Tracking substep start: steps/01-install/steps/identity (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 🔸 Service: identity
🔍 DEBUG_CHECKPOINT_07: About to execute /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/run.sh with IDENTIFIER=coordinator IDENTIFIER_PARENT=coordinator
🔍 DEBUG_CHECKPOINT_08: Running identity in AUTO mode
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] 🟢 Starting PostgreSQL provisioning for identity in sau-dev...
[INFO] Environment: identity-sau-main-dev
[INFO] Identifier: coordinator
[INFO] VM IP: 142.93.238.16
🔍 DEBUG_CHECKPOINT_A1: identity/run.sh started for SERVICE=identity
🔍 DEBUG_CHECKPOINT_A2: Checking SERVICE_ROOT: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity
🔍 DEBUG_CHECKPOINT_A3: SERVICE_ROOT exists, discovering table folders
🔍 DEBUG_CHECKPOINT_A4: Found subfolder: auth
🔍 DEBUG_CHECKPOINT_A4b: Checking for nested schema layout in: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth
🔍 DEBUG_CHECKPOINT_A4c: Found nested steps dir: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps (display: auth/login)
🔍 DEBUG_CHECKPOINT_A5: Table step dirs discovered: auth/login|/opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps
🔍 DEBUG_CHECKPOINT_A6: Checking if we have table folders to process
[INFO] 📚 Detected grouped table folders under identity/: auth/login
🔍 DEBUG_CHECKPOINT_A7: Current IDENTIFIER=coordinator
🔍 DEBUG_CHECKPOINT_A8_PROCEED: Processing tables on coordinator/main node
🔍 DEBUG_CHECKPOINT_A9: Processing table: auth/login at /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps
[INFO] 🔸 Table group: auth/login
🔍 DEBUG_CHECKPOINT_A10: About to run numbered steps for table: auth/login
🔍 DEBUG_CHECKPOINT_B1: run_all_numbered_steps_in_dir called for dir=/opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps table=auth/login
🔍 DEBUG_CHECKPOINT_B2: Found 1 numbered steps: 01-init-schema.sh
🔍 DEBUG_CHECKPOINT_B3: About to run step: 01-init-schema.sh
Ab substep 0 compelete start
[DEBUG] Tracking substep start: steps/01-install/steps/identity/auth/login/01-init-schema (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
Ab substep 0 compelete start
[INFO] 📦 01 init schema...
Ab substep 1 compelete start
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing auth.login_account table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Identifier: coordinator
Database: fastorder_identity_sau_main_dev_db
Host: db-identity-sau-main-dev-postgresql.fastorder.com:5432
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔐 Connecting to PostgreSQL over SSL (verify-full + mTLS)...
🗄️ Checking database: fastorder_identity_sau_main_dev_db
ℹ️ Database fastorder_identity_sau_main_dev_db already exists
✅ Connected to database: fastorder_identity_sau_main_dev_db
ℹ️ Checking synchronous replication configuration...
synchronous_standby_names: ''
Connected standbys: 0
ℹ️ Synchronous replication not configured (standbys will be added later)
🔧 Installing extensions...
NOTICE: extension "uuid-ossp" already exists, skipping
CREATE EXTENSION
NOTICE: extension "dblink" already exists, skipping
CREATE EXTENSION
🔧 Installing Citus extension on coordinator...
NOTICE: extension "citus" already exists, skipping
CREATE EXTENSION
✅ Citus extension installed
✅ Extensions installed
🔧 Installing UUIDv7 function...
✅ UUIDv7 function installed
🔧 Creating auth schema...
NOTICE: schema "auth" already exists, skipping
CREATE SCHEMA
✅ Schema created
🔧 Creating account_status ENUM...
DO
✅ ENUM created
🔧 Creating auth.login_account table...
CREATE TABLE
✅ Table created (Citus-compatible with region_hint in all constraints)
🔧 Creating indexes...
CREATE INDEX
CREATE INDEX
✅ Indexes created
🔧 Creating Citus REFERENCE table for CDC compatibility...
create_reference_table
------------------------
(1 row)
✅ Table created as REFERENCE table (replicated to all nodes)
CDC via Debezium will work correctly on coordinator
🎉 Schema initialization complete for fastorder_identity_sau_main_dev_db
ℹ️ Skipping LISTEN/NOTIFY trigger on coordinator
CDC via Debezium is the primary change tracking mechanism
📊 Registering environment in monitoring database (obs schema)...
Topology: /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
Resource IP: 142.93.238.16
⚠️ Could not connect to monitoring database, skipping registration
You can manually register later using:
/opt/fastorder/bash/scripts/env_app_setup/setup/04-postgresql/steps/register-authN-af-aaaa1-dev.sh
==========================================
✅ Schema initialization complete!
==========================================
Ab substep 1 compelete end
Ab substep 2 compelete start
Ab substep 2 compelete end
🔍 DEBUG_CHECKPOINT_B4: Completed step: 01-init-schema.sh
🔍 DEBUG_CHECKPOINT_B5: All numbered steps completed for /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps
🔍 DEBUG_CHECKPOINT_A11: Completed numbered steps for table: auth/login
compeleted here
🔍 DEBUG_CHECKPOINT_A12: All tables processed
End of 04-postgresql/steps/01-install/steps/identity/run.sh
[0;32m✓[0m ✅ Coordinator setup completed
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Setting up 1 worker(s) (Citus data nodes)…
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
→ Setting up worker: worker-01
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] 📁 Initializing log directories...
[2026-01-02 07:02:45 UTC] USER=unknown EUID=33 PID=1624489 ACTION=fsop ARGS=mkdir -p /var/log/fastorder/metrics
[2026-01-02 07:02:45 UTC] USER=unknown EUID=33 PID=1624496 ACTION=fsop ARGS=mkdir -p /var/log/fastorder/audit
[2026-01-02 07:02:45 UTC] USER=unknown EUID=33 PID=1624503 ACTION=fsop ARGS=mkdir -p /var/log/fastorder/provisioning
[2026-01-02 07:02:45 UTC] USER=unknown EUID=33 PID=1624510 ACTION=fsop ARGS=chmod 777 /var/log/fastorder/metrics
[2026-01-02 07:02:45 UTC] USER=unknown EUID=33 PID=1624517 ACTION=fsop ARGS=chmod 777 /var/log/fastorder/audit
[2026-01-02 07:02:45 UTC] USER=unknown EUID=33 PID=1624524 ACTION=fsop ARGS=chmod 777 /var/log/fastorder/provisioning
/opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/run.sh: line 41: ok: command not found
[INFO] 🟢 Starting PostgreSQL provisioning for identity in sau-dev...
[INFO] Environment: identity-sau-main-dev
[INFO] Identifier: worker-01
[INFO] VM IP: 142.93.238.16
[DEBUG] RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3 JOB_UUID=438ff2f1-77b4-4b8d-aa09-3d1698934d76
[DEBUG] Tracking substep start: steps/01-install/steps/00-configure-network-hosts (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 00 configure network hosts...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[WARN] Could not find PostgreSQL IP for worker-01 in topology.json, allocating new VM IP...
[INFO] Allocated new VM IP: 10.100.1.214 for db-worker-01-postgresql
[INFO] ═══════════════════════════════════════════════════════════════
[INFO] CONFIGURING POSTGRESQL NETWORK & DNS
[INFO] ═══════════════════════════════════════════════════════════════
[INFO] Environment: identity-sau-main-dev
[INFO] Identifier: worker-01
[INFO] PostgreSQL IP: 10.100.1.214
[INFO] Primary hostname: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
[INFO] Adding /etc/hosts entry for worker-01...
[INFO] db-identity-sau-main-dev-postgresql-worker-01.fastorder.com → 10.100.1.214
[INFO] ➕ Adding db-identity-sau-main-dev-postgresql-worker-01.fastorder.com → 10.100.1.214
[0;32m✅[0m ✅ Added: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com → 10.100.1.214
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;32m✅[0m ✅ Network & DNS configuration complete
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[INFO] Verifying /etc/hosts entries:
10.100.1.214 db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
[DEBUG] Tracking substep start: steps/01-install/steps/01-prepare-ssl-server-postgres (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 01 prepare ssl server postgres...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
📦 PostgreSQL Server Certificate Generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau (Saudi Arabia)
Branch: main
Env: dev
Node: worker-01
Primary CN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Alt CN: identity-sau-main-dev.fastorder.com
VM IP: 142.93.238.16
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ Removing existing server certificates (preserving client certs)...
[2026-01-02 07:02:50 UTC] USER=www-data EUID=0 PID=1625132 ACTION=fsop ARGS=rm -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
✅ Ensuring directories exist: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01 and /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🔐 Generating 4096-bit private key...
[2026-01-02 07:02:50 UTC] USER=www-data EUID=0 PID=1625153 ACTION=fsop ARGS=chmod 755 /tmp/pg-cert-gen-1625001
[2026-01-02 07:02:50 UTC] USER=www-data EUID=0 PID=1625162 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-cert-gen-1625001/ra_root.crt
[2026-01-02 07:02:50 UTC] USER=www-data EUID=0 PID=1625171 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-cert-gen-1625001/ra_root.key
[2026-01-02 07:02:50 UTC] USER=www-data EUID=0 PID=1625180 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1625001/ra_root.crt
[2026-01-02 07:02:50 UTC] USER=www-data EUID=0 PID=1625189 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1625001/ra_root.key
📝 Creating certificate signing request (CSR)...
📜 Signing certificate with internal CA...
Certificate request self-signature ok
subject=C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
[2026-01-02 07:02:56 UTC] USER=www-data EUID=0 PID=1625234 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1625001/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
[2026-01-02 07:02:56 UTC] USER=www-data EUID=0 PID=1625243 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1625001/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
[2026-01-02 07:02:56 UTC] USER=www-data EUID=0 PID=1625252 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
📋 Setting up CA certificate...
[2026-01-02 07:02:56 UTC] USER=www-data EUID=0 PID=1625261 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1625001/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:02:56 UTC] USER=www-data EUID=0 PID=1625270 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:02:56 UTC] USER=www-data EUID=0 PID=1625279 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:02:56 UTC] USER=www-data EUID=0 PID=1625288 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
✅ Using CA certificate: /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt
🚚 Setting up key in private directory...
Key already in correct location (CERT_DIR == KEY_DIR)
🔒 Securing key and cert permissions...
[2026-01-02 07:02:56 UTC] USER=www-data EUID=0 PID=1625299 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625308 ACTION=fsop ARGS=chmod 600 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625317 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625326 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625335 ACTION=fsop ARGS=chown root:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625344 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🔍 Verifying certificate...
Certificate details:
Subject: C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
--
X509v3 Subject Alternative Name:
DNS:db-identity-sau-main-dev-postgresql-worker-01.fastorder.com, DNS:identity-sau-main-dev.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-worker-01.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-worker-01, DNS:localhost, IP Address:142.93.238.16, IP Address:127.0.0.1
X509v3 Subject Key Identifier:
⚠️ Certificate chain verification: FAILED (but certificate may still work)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ PostgreSQL Server Certificate Generated Successfully!
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Node: worker-01
Primary CN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Certificate files installed:
📜 Server cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
🔑 Server key: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
🏛️ CA cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt (ca.crt symlink also available)
To use these certificates in PostgreSQL:
1. Update postgresql.conf:
ssl = on
ssl_cert_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt'
ssl_key_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key'
ssl_ca_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt'
2. Restart PostgreSQL:
command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh passthru systemctl restart postgresql@identity-sau-main-dev-worker-01.service
3. Test SSL connection:
psql "host=db-identity-sau-main-dev-postgresql-worker-01.fastorder.com port=5432 user=postgres sslmode=verify-full"
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: postgres
Identifier: worker-01
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: worker-01
User (CN): postgres
Hostname: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625400 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-worker-01-postgres
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625409 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-worker-01-postgres/ra_root.crt
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625418 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-worker-01-postgres/ra_root.key
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625427 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-postgres/ra_root.crt
[2026-01-02 07:02:57 UTC] USER=www-data EUID=0 PID=1625436 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-postgres/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = postgres
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625450 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625459 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625468 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/postgres.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625477 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625486 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625495 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625504 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/postgres.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625513 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/postgres_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625522 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625531 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625540 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625549 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625558 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625567 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625576 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625585 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625611 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625620 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625629 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625638 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625647 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:58 UTC] USER=www-data EUID=0 PID=1625656 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625665 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625674 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625683 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625692 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625701 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625711 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625721 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625730 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625739 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625748 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625757 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625766 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625775 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625784 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625793 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625802 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625811 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625821 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625831 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625840 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625849 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625858 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625867 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625876 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625885 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625894 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625903 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625912 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625921 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:02:59 UTC] USER=www-data EUID=0 PID=1625931 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1625941 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1625950 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1625959 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1625968 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1625977 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1625986 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1625995 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626004 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626013 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626022 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626031 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626041 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626051 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626061 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626070 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626079 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626088 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626097 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626106 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626115 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:03:00 UTC] USER=www-data EUID=0 PID=1626124 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: postgres
Node: worker-01
FQDN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-worker-01.fastorder.com -U postgres -d postgres
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: postgres
Identifier: worker-01
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: worker-01
User (CN): postgres
Hostname: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:03:01 UTC] USER=www-data EUID=0 PID=1626165 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-worker-01-postgres
[2026-01-02 07:03:01 UTC] USER=www-data EUID=0 PID=1626174 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-worker-01-postgres/ra_root.crt
[2026-01-02 07:03:01 UTC] USER=www-data EUID=0 PID=1626183 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-worker-01-postgres/ra_root.key
[2026-01-02 07:03:01 UTC] USER=www-data EUID=0 PID=1626192 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-postgres/ra_root.crt
[2026-01-02 07:03:01 UTC] USER=www-data EUID=0 PID=1626201 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-postgres/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = postgres
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626227 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626236 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626245 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/postgres.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626254 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626263 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626272 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626281 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/postgres.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626290 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-postgres/postgres_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626299 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626308 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626317 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626328 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626337 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626346 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626355 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626364 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626373 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626382 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626408 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626417 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626426 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626435 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626444 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626453 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:03:02 UTC] USER=www-data EUID=0 PID=1626462 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626471 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626480 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626489 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626498 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626508 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626518 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626527 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626536 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626545 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626554 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626563 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626572 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626581 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626590 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626599 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626608 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626618 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626628 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626637 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626646 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626655 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626664 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626673 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626682 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626691 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626700 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626709 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:03:03 UTC] USER=www-data EUID=0 PID=1626718 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626728 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626738 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626747 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626756 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626765 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626774 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626783 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626792 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626801 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626810 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626819 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key.pkcs1
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626828 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/postgres_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres_der.key
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626838 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626848 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626857 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626866 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626875 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626884 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626893 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626902 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626911 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:03:04 UTC] USER=www-data EUID=0 PID=1626920 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: postgres
Node: worker-01
FQDN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-worker-01.fastorder.com -U postgres -d postgres
[DEBUG] Tracking substep start: steps/01-install/steps/02-setup-pg-instance (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 02 setup pg instance...
[0;34m[DEADLOCK-PREVENTION][0m Deadlock prevention library loaded
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
🔑 Configuring AWS credentials...
✅ Using permanent AWS credentials from /var/www/.aws/credentials
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔑 Configuring AWS credentials...
[WARN] ~/.aws/credentials file not found
[WARN] AWS operations may require SSO login
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] Using existing db-worker-01-postgresql environment: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com (10.100.1.214)
[INFO] PostgreSQL will listen on application-specific IP: 10.100.1.214
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: worker-01
[0;34m[INFO][0m Data dir: /var/lib/postgresql/17/identity-sau-main-dev/worker-01
[0;34m[INFO][0m Port: 5432
[0;34m[INFO][0m Hostname: db-identity-sau-main-dev-postgresql-worker-01
[2026-01-02 07:03:06 UTC] USER=www-data EUID=0 PID=1627014 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:03:07 UTC] USER=www-data EUID=0 PID=1627035 ACTION=fsop ARGS=chmod 755 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:03:07 UTC] USER=www-data EUID=0 PID=1627056 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:03:07 UTC] USER=www-data EUID=0 PID=1627077 ACTION=fsop ARGS=chown root:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[1;33m[WARN][0m Server certificate not found at /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
[0;34m[INFO][0m Generating server certificate using ssl/server.sh...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
📦 PostgreSQL Server Certificate Generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau (Saudi Arabia)
Branch: main
Env: dev
Node: worker-01
Primary CN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Alt CN: identity-sau-main-dev.fastorder.com
VM IP: 142.93.238.16
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ Removing existing server certificates (preserving client certs)...
[2026-01-02 07:03:07 UTC] USER=www-data EUID=0 PID=1627119 ACTION=fsop ARGS=rm -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
✅ Ensuring directories exist: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01 and /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:03:07 UTC] USER=www-data EUID=0 PID=1627128 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🔐 Generating 4096-bit private key...
[2026-01-02 07:03:07 UTC] USER=www-data EUID=0 PID=1627138 ACTION=fsop ARGS=chmod 755 /tmp/pg-cert-gen-1627084
[2026-01-02 07:03:07 UTC] USER=www-data EUID=0 PID=1627147 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-cert-gen-1627084/ra_root.crt
[2026-01-02 07:03:07 UTC] USER=www-data EUID=0 PID=1627156 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-cert-gen-1627084/ra_root.key
[2026-01-02 07:03:08 UTC] USER=www-data EUID=0 PID=1627165 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1627084/ra_root.crt
[2026-01-02 07:03:08 UTC] USER=www-data EUID=0 PID=1627174 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1627084/ra_root.key
📝 Creating certificate signing request (CSR)...
📜 Signing certificate with internal CA...
Certificate request self-signature ok
subject=C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627211 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1627084/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627220 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1627084/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627229 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
📋 Setting up CA certificate...
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627238 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1627084/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627247 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627256 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627265 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
✅ Using CA certificate: /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt
🚚 Setting up key in private directory...
Key already in correct location (CERT_DIR == KEY_DIR)
🔒 Securing key and cert permissions...
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627276 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627285 ACTION=fsop ARGS=chmod 600 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627294 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
[2026-01-02 07:03:10 UTC] USER=www-data EUID=0 PID=1627303 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
[2026-01-02 07:03:11 UTC] USER=www-data EUID=0 PID=1627312 ACTION=fsop ARGS=chown root:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:03:11 UTC] USER=www-data EUID=0 PID=1627321 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🔍 Verifying certificate...
Certificate details:
Subject: C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
--
X509v3 Subject Alternative Name:
DNS:db-identity-sau-main-dev-postgresql-worker-01.fastorder.com, DNS:identity-sau-main-dev.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-worker-01.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-worker-01, DNS:localhost, IP Address:142.93.238.16, IP Address:127.0.0.1
X509v3 Subject Key Identifier:
⚠️ Certificate chain verification: FAILED (but certificate may still work)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ PostgreSQL Server Certificate Generated Successfully!
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Node: worker-01
Primary CN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Certificate files installed:
📜 Server cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
🔑 Server key: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
🏛️ CA cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt (ca.crt symlink also available)
To use these certificates in PostgreSQL:
1. Update postgresql.conf:
ssl = on
ssl_cert_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt'
ssl_key_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key'
ssl_ca_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt'
2. Restart PostgreSQL:
command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh passthru systemctl restart postgresql@identity-sau-main-dev-worker-01.service
3. Test SSL connection:
psql "host=db-identity-sau-main-dev-postgresql-worker-01.fastorder.com port=5432 user=postgres sslmode=verify-full"
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m ✅ Using canonical certificate path (hardened, ProtectHome=true compatible)
[2026-01-02 07:03:11 UTC] USER=www-data EUID=0 PID=1627350 ACTION=fsop ARGS=test -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.crt
[2026-01-02 07:03:11 UTC] USER=www-data EUID=0 PID=1627359 ACTION=fsop ARGS=test -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/server.key
[2026-01-02 07:03:11 UTC] USER=www-data EUID=0 PID=1627368 ACTION=fsop ARGS=test -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
[0;32m[OK][0m mTLS certificates OK (server cert + client certs verified) and keys secured
[0;34m[INFO][0m Preflight: stopping any conflicting Postgres services/processes on port 5432…
[2026-01-02 07:03:11 UTC] USER=www-data EUID=0 PID=1627389 ACTION=passthru ARGS=systemctl stop postgresql@identity-sau-main-dev-worker-01.service
[2026-01-02 07:03:11 UTC] USER=www-data EUID=0 PID=1627410 ACTION=passthru ARGS=systemctl stop postgresql
[0;32m[OK][0m No conflicting Postgres left on port 5432
[0;32m[OK][0m Generated new postgres password for initdb
[2026-01-02 07:03:37 UTC] USER=www-data EUID=0 PID=1627685 ACTION=fsop ARGS=chown postgres:postgres /tmp/.pg_pwfile.mr99qx
[2026-01-02 07:03:37 UTC] USER=www-data EUID=0 PID=1627706 ACTION=fsop ARGS=chmod 600 /tmp/.pg_pwfile.mr99qx
[2026-01-02 07:03:37 UTC] USER=www-data EUID=0 PID=1627730 ACTION=fsop ARGS=mkdir -p /var/lib/postgresql/17/identity-sau-main-dev
[2026-01-02 07:03:37 UTC] USER=www-data EUID=0 PID=1627752 ACTION=fsop ARGS=chown postgres:postgres /var/lib/postgresql/17/identity-sau-main-dev
[2026-01-02 07:03:37 UTC] USER=www-data EUID=0 PID=1627774 ACTION=fsop ARGS=chmod 755 /var/lib/postgresql/17/identity-sau-main-dev
[0;34m[INFO][0m Initializing cluster in /var/lib/postgresql/17/identity-sau-main-dev/worker-01 (SCRAM; pwfile)
[2026-01-02 07:03:37 UTC] USER=www-data EUID=0 PID=1627796 ACTION=fsop ARGS=mkdir -p /var/lib/postgresql/17/identity-sau-main-dev/worker-01
[2026-01-02 07:03:37 UTC] USER=www-data EUID=0 PID=1627817 ACTION=fsop ARGS=chown postgres:postgres /var/lib/postgresql/17/identity-sau-main-dev/worker-01
[2026-01-02 07:03:38 UTC] USER=www-data EUID=0 PID=1627838 ACTION=fsop ARGS=chmod 700 /var/lib/postgresql/17/identity-sau-main-dev/worker-01
[2026-01-02 07:03:38 UTC] USER=www-data EUID=0 PID=1627859 ACTION=fsop ARGS=mkdir -p /var/run/postgresql-identity-sau-main-dev-worker-01
[2026-01-02 07:03:38 UTC] USER=www-data EUID=0 PID=1627920 ACTION=fsop ARGS=chown postgres:postgres /var/run/postgresql-identity-sau-main-dev-worker-01
[2026-01-02 07:03:38 UTC] USER=www-data EUID=0 PID=1627996 ACTION=fsop ARGS=chmod 2775 /var/run/postgresql-identity-sau-main-dev-worker-01
[2026-01-02 07:03:38 UTC] USER=www-data EUID=0 PID=1628008 ACTION=passthru ARGS=sudo -u postgres /usr/lib/postgresql/17/bin/initdb -D /var/lib/postgresql/17/identity-sau-main-dev/worker-01 --locale=en_US.UTF-8 --encoding=UTF8 --auth-local=scram-sha-256 --auth-host=scram-sha-256 --pwfile=/tmp/.pg_pwfile.mr99qx
The files belonging to this database system will be owned by user "postgres".
This user must also own the server process.
The database cluster will be initialized with locale "en_US.UTF-8".
The default text search configuration will be set to "english".
Data page checksums are disabled.
fixing permissions on existing directory /var/lib/postgresql/17/identity-sau-main-dev/worker-01 ... ok
creating subdirectories ... ok
selecting dynamic shared memory implementation ... posix
selecting default "max_connections" ... 100
selecting default "shared_buffers" ... 128MB
selecting default time zone ... Etc/UTC
creating configuration files ... ok
running bootstrap script ... ok
performing post-bootstrap initialization ... ok
syncing data to disk ... ok
Success. You can now start the database server using:
/usr/lib/postgresql/17/bin/pg_ctl -D /var/lib/postgresql/17/identity-sau-main-dev/worker-01 -l logfile start
[0;32m[OK][0m initdb complete
[2026-01-02 07:03:39 UTC] USER=www-data EUID=0 PID=1628045 ACTION=fsop ARGS=rm -f /tmp/.pg_pwfile.mr99qx
[0;34m[INFO][0m Writing postgresql.conf (TLS≥1.2, SCRAM, audit logs)
[0;32m[OK][0m postgresql.conf updated successfully
[0;34m[INFO][0m Writing pg_hba.conf (mTLS with client certificates + SCRAM, least-privilege)
[2026-01-02 07:03:40 UTC] USER=www-data EUID=0 PID=1628094 ACTION=fsop ARGS=cp /tmp/tmp.0Y915elaTw /var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
[2026-01-02 07:03:40 UTC] USER=www-data EUID=0 PID=1628115 ACTION=fsop ARGS=chown postgres:postgres /var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
[2026-01-02 07:03:40 UTC] USER=www-data EUID=0 PID=1628136 ACTION=fsop ARGS=chmod 600 /var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
[0;32m[OK][0m pg_hba.conf updated
[0;34m[INFO][0m Creating systemd unit: /etc/systemd/system/postgresql@identity-sau-main-dev-worker-01.service
[2026-01-02 07:03:40 UTC] USER=www-data EUID=0 PID=1628161 ACTION=fsop ARGS=mv -f /tmp/.pg_unit.sezAUl /etc/systemd/system/postgresql@identity-sau-main-dev-worker-01.service
[2026-01-02 07:03:40 UTC] USER=www-data EUID=0 PID=1628182 ACTION=fsop ARGS=chmod 0644 /etc/systemd/system/postgresql@identity-sau-main-dev-worker-01.service
[0;32m[OK][0m systemd unit written
[2026-01-02 07:03:40 UTC] USER=www-data EUID=0 PID=1628203 ACTION=fsop ARGS=mkdir -p /var/lib/pgbackrest /var/spool/pgbackrest /var/log/pgbackrest
[2026-01-02 07:03:40 UTC] USER=www-data EUID=0 PID=1628225 ACTION=fsop ARGS=chown postgres:postgres /var/lib/pgbackrest /var/spool/pgbackrest /var/log/pgbackrest
[2026-01-02 07:03:40 UTC] USER=www-data EUID=0 PID=1628246 ACTION=passthru ARGS=systemctl daemon-reload
[0;34m[INFO][0m Starting PostgreSQL instance...
[2026-01-02 07:03:42 UTC] USER=www-data EUID=0 PID=1628362 ACTION=passthru ARGS=systemctl start postgresql@identity-sau-main-dev-worker-01.service
[0;34m[INFO][0m Waiting for ACTIVE (systemd)…
[2026-01-02 07:03:43 UTC] USER=www-data EUID=0 PID=1628403 ACTION=passthru ARGS=systemctl is-active --quiet postgresql@identity-sau-main-dev-worker-01.service
[0;32m[OK][0m Service ACTIVE
[0;34m[INFO][0m Waiting for port 5432 bind…
[0;32m[OK][0m Port bound
[0;34m[INFO][0m Waiting pg_isready (socket)…
[0;32m[OK][0m Readiness via socket OK
[0;34m[INFO][0m Waiting pg_isready (TCP db-identity-sau-main-dev-postgresql-worker-01.fastorder.com:5432)…
[0;32m[OK][0m Startup sequence complete
[0;34m[INFO][0m Validating core security GUCs (via local socket)…
[0;32m[OK][0m Security GUCs verified (ssl, min TLS, SCRAM, audit logs)
[0;34m[INFO][0m Provisioning application database and Debezium role (if not exists)...
[0;34m[INFO][0m Checking if database fastorder_identity_sau_main_dev_db exists...
[0;34m[INFO][0m DB check result: exit_code=0, output='[2026-01-02 07:03:44 UTC] USER=www-data EUID=0 PID=1628559 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -Atqc SELECT 1 FROM pg_database WHERE datname = 'fastorder_identity_sau_main_dev_db''
[0;34m[INFO][0m Creating database fastorder_identity_sau_main_dev_db...
[2026-01-02 07:03:44 UTC] USER=www-data EUID=0 PID=1628582 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c CREATE DATABASE "fastorder_identity_sau_main_dev_db" ENCODING 'UTF8' LC_COLLATE 'en_US.UTF-8' LC_CTYPE 'en_US.UTF-8' TEMPLATE template0;
CREATE DATABASE
[0;32m[OK][0m Database fastorder_identity_sau_main_dev_db created
[0;34m[INFO][0m Checking if role debezium_user exists...
[0;34m[INFO][0m Role check result: exit_code=0, output='[2026-01-02 07:03:45 UTC] USER=www-data EUID=0 PID=1628606 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -Atqc SELECT 1 FROM pg_roles WHERE rolname = 'debezium_user''
[0;34m[INFO][0m Creating role debezium_user...
[2026-01-02 07:03:45 UTC] USER=www-data EUID=0 PID=1628634 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c CREATE ROLE debezium_user LOGIN PASSWORD 'OGyPS36E5SRw7wkShQNgoI+F';
CREATE ROLE
[0;32m[OK][0m Role debezium_user created
[2026-01-02 07:03:45 UTC] USER=www-data EUID=0 PID=1628657 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c GRANT CONNECT ON DATABASE "fastorder_identity_sau_main_dev_db" TO debezium_user;
GRANT
[0;32m[OK][0m Application DB (fastorder_identity_sau_main_dev_db) + Debezium role (debezium_user) provisioned (idempotent)
[0;34m[INFO][0m Applying connection and memory optimizations...
[0;34m[INFO][0m Current settings: max_connections=100, work_mem=4MB
[0;34m[INFO][0m Target settings (worker): max_connections=100, work_mem=8MB
[2026-01-02 07:03:46 UTC] USER=www-data EUID=0 PID=1628734 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c ALTER SYSTEM SET max_connections = 100;
ALTER SYSTEM
[2026-01-02 07:03:46 UTC] USER=www-data EUID=0 PID=1628757 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c ALTER SYSTEM SET work_mem = '8MB';
ALTER SYSTEM
[2026-01-02 07:03:46 UTC] USER=www-data EUID=0 PID=1628780 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -c SELECT pg_reload_conf();
pg_reload_conf
----------------
t
(1 row)
[0;32m[OK][0m Settings applied to postgresql.auto.conf
[2026-01-02 07:03:46 UTC] USER=www-data EUID=0 PID=1628796 ACTION=passthru ARGS=sudo -u postgres test -f /var/lib/postgresql/17/identity-sau-main-dev/worker-01/standby.signal
[0;34m[INFO][0m Service recently started (3s ago) - restarting to apply max_connections...
[0;34m[INFO][0m Stopping service...
[2026-01-02 07:03:46 UTC] USER=www-data EUID=0 PID=1628818 ACTION=passthru ARGS=systemctl stop postgresql@identity-sau-main-dev-worker-01.service
[0;34m[INFO][0m Waiting for port 5432 to be released...
[0;32m[OK][0m Port 5432 released
[0;34m[INFO][0m Starting service...
[2026-01-02 07:03:50 UTC] USER=www-data EUID=0 PID=1628858 ACTION=passthru ARGS=systemctl start postgresql@identity-sau-main-dev-worker-01.service
[2026-01-02 07:03:56 UTC] USER=www-data EUID=0 PID=1629002 ACTION=passthru ARGS=systemctl is-active --quiet postgresql@identity-sau-main-dev-worker-01.service
[0;32m[OK][0m ✅ Optimization complete: max_connections=100, work_mem=8MB
[0;32m[OK][0m Synchronous replication already configured (synchronous_commit: on)
[0;34m[INFO][0m Setting postgres password via centralized script... for worker-01
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
⚠️ ~/.aws/credentials file not found
⚠️ Using environment-based AWS authentication
[0;34m╔════════════════════════════════════════════════════════════╗[0m
[0;34m║ PostgreSQL Password Rotation via AWS Secrets Manager ║[0m
[0;34m╚════════════════════════════════════════════════════════════╝[0m
[0;34mEnvironment Configuration:[0m
[0;34m Service: identity[0m
[0;34m Zone: sau[0m
[0;34m Environment: dev[0m
[0;34m Identifier: worker-01[0m
[0;34mAWS Secret: fastorder/db/identity/sau/main/dev/postgresql/worker-01[0m
[0;34mConnection Info:[0m
[0;34m Socket Dir: /var/run/postgresql-identity-sau-main-dev-worker-01[0m
[0;34m Port: 5432[0m
[0;34mTesting AWS Secrets Manager connectivity...[0m
ℹ️ Testing AWS IAM credentials...
✅ AWS IAM credentials are valid
{
"UserId": "AIDAWYLM4MSHFSCGU7QUM",
"Account": "464621692046",
"Arn": "arn:aws:iam::464621692046:user/fo-dev"
}
[0;34mMethod 1 (PREFERRED): AWS Secrets Manager Rotation[0m
[0;34m────────────────────────────────────────────────────────────[0m
[0;34mThis method uses AWS Secrets Manager's built-in rotation:[0m
[0;34m ✓ Zero-downtime (dual-password window)[0m
[0;34m ✓ Automatic rollback on failure[0m
[0;34m ✓ CloudTrail audit log[0m
[0;34m ✓ CloudWatch metrics[0m
[0;34m ✓ No secret exposure in scripts[0m
[0;34mNon-interactive mode: Proceeding with password rotation automatically[0m
[0;34mInitial setup: Using password from initdb[0m
[0;32m✓ PostgreSQL password already set during initdb[0m
[0;34mStoring password in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/worker-01[0m
ℹ️ Setting PostgreSQL credentials in vault: fastorder/db/identity/sau/main/dev/postgresql/worker-01
ℹ️ Setting secret in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/worker-01
✅ Secret created: fastorder/db/identity/sau/main/dev/postgresql/worker-01
✅ PostgreSQL credentials set in vault: fastorder/db/identity/sau/main/dev/postgresql/worker-01
[0;32m✓ Password stored in AWS Secrets Manager[0m
[0;34mVerifying new credentials...[0m
[0;32m✓ New credentials retrieved from AWS Secrets Manager[0m
[0;34mTesting PostgreSQL connection with new credentials...[0m
[0;32m✓ PostgreSQL connection successful (socket authentication)[0m
[0;32m✓ ╔════════════════════════════════════════════════════════════╗[0m
[0;32m✓ ║ Password Rotation Complete! ║[0m
[0;32m✓ ╚════════════════════════════════════════════════════════════╝[0m
[0;34mSecret: fastorder/db/identity/sau/main/dev/postgresql/worker-01[0m
[0;34mMethod: Direct Update (stored in AWS Secrets Manager)[0m
[0;34mStatus: Completed[0m
[0;34mTo retrieve credentials:[0m
[0;34m # Using Bash library[0m
[0;34m source /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh[0m
[0;34m get_pg_credentials worker-01[0m
[0;34mAudit trail: AWS CloudTrail (for Secrets Manager operations)[0m
[0;32m✓ Done![0m
[0;32m[OK][0m Password set and persisted
[0;34m[INFO][0m Updating /etc/hosts with PostgreSQL hostname mappings...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m CONFIGURING POSTGRESQL NETWORK & DNS
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: worker-01
[0;34m[INFO][0m PostgreSQL IP: 10.100.1.214
[0;34m[INFO][0m Primary hostname: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
[0;34m[INFO][0m Adding /etc/hosts entry for worker-01...
[0;34m[INFO][0m db-identity-sau-main-dev-postgresql-worker-01.fastorder.com → 10.100.1.214
[0;34m[INFO][0m ✅ db-identity-sau-main-dev-postgresql-worker-01.fastorder.com already exists with correct IP
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;32m✅[0m ✅ Network & DNS configuration complete
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Verifying /etc/hosts entries:
10.100.1.214 db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
[0;32m[OK][0m PostgreSQL 'identity-sau-main-dev' is up with TLS/SCRAM/logging.
Superuser TCP mode: cert
Connect (mTLS):
psql "sslmode=verify-full sslrootcert=/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt \
sslcert=/home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.crt \
sslkey=/home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/postgres.key \
host=db-identity-sau-main-dev-postgresql-worker-01 port=5432 dbname=postgres user=postgres"
File been compeleted perfectly: 02-setup-pg-instance
[0;34m[INFO][0m Registering PostgreSQL node to observability API...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: PostgreSQL
[INFO] Identifier: identity-sau-main-dev-postgresql-worker-01
[INFO] Identifier Parent: worker-01
[INFO] IP: 10.100.1.214
[INFO] Port: 5432
[INFO] FQDN: db-identity-sau-main-dev-postgresql-worker-01
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: 2a8d7237-0c1b-4286-8ffc-cd46f4f7052e
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[0;32m[OK][0m PostgreSQL node registered to observability API
[DEBUG] Tracking substep start: steps/01-install/steps/03-role (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 03 role...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[2026-01-02 07:04:09 UTC] USER=www-data EUID=0 PID=1629359 ACTION=fsop ARGS=test -f /var/lib/postgresql/17/identity-sau-main-dev/worker-01/standby.signal
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: debezium_user
Identifier: worker-01
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: worker-01
User (CN): debezium_user
Hostname: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:04:33 UTC] USER=www-data EUID=0 PID=1629650 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-worker-01-debezium_user
[2026-01-02 07:04:33 UTC] USER=www-data EUID=0 PID=1629659 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-worker-01-debezium_user/ra_root.crt
[2026-01-02 07:04:33 UTC] USER=www-data EUID=0 PID=1629668 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-worker-01-debezium_user/ra_root.key
[2026-01-02 07:04:33 UTC] USER=www-data EUID=0 PID=1629677 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-debezium_user/ra_root.crt
[2026-01-02 07:04:33 UTC] USER=www-data EUID=0 PID=1629686 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-debezium_user/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = debezium_user
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:34 UTC] USER=www-data EUID=0 PID=1629722 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:34 UTC] USER=www-data EUID=0 PID=1629731 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:34 UTC] USER=www-data EUID=0 PID=1629740 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-debezium_user/debezium_user.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629749 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-debezium_user/debezium_user.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.crt
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629758 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-debezium_user/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629767 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629776 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-debezium_user/debezium_user.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629785 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-debezium_user/debezium_user_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user_der.key
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629794 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629803 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629812 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629821 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629831 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629840 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user_der.key
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629849 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629858 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629884 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629893 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629902 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629911 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629920 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629929 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629938 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629947 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:35 UTC] USER=www-data EUID=0 PID=1629956 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1629965 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1629974 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user_der.key
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1629984 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1629994 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630003 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630012 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630021 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630030 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630039 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630048 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630057 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630066 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630075 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630084 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user_der.key
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630094 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630104 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630113 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630122 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630131 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630140 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630149 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630158 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630167 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630176 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630185 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1
[2026-01-02 07:04:36 UTC] USER=www-data EUID=0 PID=1630194 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user_der.key
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630204 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630214 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630223 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630232 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630241 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630250 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630259 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630268 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630277 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630286 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630295 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key.pkcs1
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630304 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/debezium_user_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user_der.key
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630314 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630324 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630335 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630344 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630353 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630362 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630371 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630380 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630389 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:04:37 UTC] USER=www-data EUID=0 PID=1630398 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: debezium_user
Node: worker-01
FQDN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/debezium_user.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-worker-01.fastorder.com -U debezium_user -d postgres
🔐 Generating replicator client certificate for worker-01...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: replicator
Identifier: worker-01
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: worker-01
User (CN): replicator
Hostname: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:04:38 UTC] USER=www-data EUID=0 PID=1630535 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-worker-01-replicator
[2026-01-02 07:04:38 UTC] USER=www-data EUID=0 PID=1630546 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-worker-01-replicator/ra_root.crt
[2026-01-02 07:04:38 UTC] USER=www-data EUID=0 PID=1630555 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-worker-01-replicator/ra_root.key
[2026-01-02 07:04:38 UTC] USER=www-data EUID=0 PID=1630565 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-replicator/ra_root.crt
[2026-01-02 07:04:38 UTC] USER=www-data EUID=0 PID=1630574 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-replicator/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = replicator
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:38 UTC] USER=www-data EUID=0 PID=1630588 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:38 UTC] USER=www-data EUID=0 PID=1630597 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:38 UTC] USER=www-data EUID=0 PID=1630606 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:04:38 UTC] USER=www-data EUID=0 PID=1630615 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630624 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630633 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630642 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630651 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630660 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630669 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630678 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630687 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630696 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630705 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630714 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630723 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630732 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630741 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630767 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630776 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630785 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630794 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630803 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630812 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630822 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630831 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630840 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630849 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:04:39 UTC] USER=www-data EUID=0 PID=1630858 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630868 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630878 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630887 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630896 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630905 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630914 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630923 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630932 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630941 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630950 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630959 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630968 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630978 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1630988 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631006 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631015 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631024 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631033 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631042 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631051 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631060 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631069 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631078 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631088 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:40 UTC] USER=www-data EUID=0 PID=1631098 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631108 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631117 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631126 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631135 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631144 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631153 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631162 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631171 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631180 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631189 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631199 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631209 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631218 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631227 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631236 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631245 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631254 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631263 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631272 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:04:41 UTC] USER=www-data EUID=0 PID=1631281 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: replicator
Node: worker-01
FQDN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-worker-01.fastorder.com -U replicator -d postgres
✅ Replicator certificate generated for worker-01
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
📦 Start executing 03-create-role.sh
📦 Setting password for user: fastorder_admin_gd
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
⚠️ ~/.aws/credentials file not found
⚠️ Using environment-based AWS authentication
[0;34m╔════════════════════════════════════════════════════════════╗[0m
[0;34m║ PostgreSQL Password Rotation via AWS Secrets Manager ║[0m
[0;34m╚════════════════════════════════════════════════════════════╝[0m
[0;34mEnvironment Configuration:[0m
[0;34m Service: identity[0m
[0;34m Zone: sau[0m
[0;34m Environment: dev[0m
[0;34m Identifier: worker-01[0m
[0;34mAWS Secret: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd[0m
[0;34mConnection Info:[0m
[0;34m Socket Dir: /var/run/postgresql-identity-sau-main-dev-worker-01[0m
[0;34m Port: 5432[0m
[0;34mTesting AWS Secrets Manager connectivity...[0m
ℹ️ Testing AWS IAM credentials...
✅ AWS IAM credentials are valid
{
"UserId": "AIDAWYLM4MSHFSCGU7QUM",
"Account": "464621692046",
"Arn": "arn:aws:iam::464621692046:user/fo-dev"
}
[0;34mMethod 1 (PREFERRED): AWS Secrets Manager Rotation[0m
[0;34m────────────────────────────────────────────────────────────[0m
[0;34mThis method uses AWS Secrets Manager's built-in rotation:[0m
[0;34m ✓ Zero-downtime (dual-password window)[0m
[0;34m ✓ Automatic rollback on failure[0m
[0;34m ✓ CloudTrail audit log[0m
[0;34m ✓ CloudWatch metrics[0m
[0;34m ✓ No secret exposure in scripts[0m
[0;34mNon-interactive mode: Proceeding with password rotation automatically[0m
[0;34mGenerating new secure password...[0m
[0;34mUser fastorder_admin_gd does not exist yet - skipping ALTER, will be created by calling script[0m
[0;32m✓ Password generated for new user: fastorder_admin_gd[0m
[0;34mStoring password in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd[0m
ℹ️ Setting PostgreSQL credentials in vault: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd
ℹ️ Setting secret in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd
✅ Secret created: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd
✅ PostgreSQL credentials set in vault: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd
[0;32m✓ Password stored in AWS Secrets Manager[0m
[0;34mVerifying new credentials...[0m
[0;32m✓ New credentials retrieved from AWS Secrets Manager[0m
[0;34mTesting PostgreSQL connection with new credentials...[0m
[0;32m✓ PostgreSQL connection successful (socket authentication)[0m
[0;32m✓ ╔════════════════════════════════════════════════════════════╗[0m
[0;32m✓ ║ Password Rotation Complete! ║[0m
[0;32m✓ ╚════════════════════════════════════════════════════════════╝[0m
[0;34mSecret: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd[0m
[0;34mMethod: Direct Update (stored in AWS Secrets Manager)[0m
[0;34mStatus: Completed[0m
[0;34mTo retrieve credentials:[0m
[0;34m # Using Bash library[0m
[0;34m source /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh[0m
[0;34m get_pg_credentials worker-01[0m
[0;34mAudit trail: AWS CloudTrail (for Secrets Manager operations)[0m
[0;32m✓ Done![0m
🔍 Retrieving password from vault with identifier: worker-01/fastorder_admin_gd
✓ Retrieved password from centralized secrets vault
🌐 Using PostgreSQL host: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: fastorder_admin_gd
Identifier: worker-01
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: worker-01
User (CN): fastorder_admin_gd
Hostname: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:04:57 UTC] USER=www-data EUID=0 PID=1631738 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-worker-01-fastorder_admin_gd
[2026-01-02 07:04:57 UTC] USER=www-data EUID=0 PID=1631749 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-worker-01-fastorder_admin_gd/ra_root.crt
[2026-01-02 07:04:57 UTC] USER=www-data EUID=0 PID=1631758 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-worker-01-fastorder_admin_gd/ra_root.key
[2026-01-02 07:04:57 UTC] USER=www-data EUID=0 PID=1631768 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-fastorder_admin_gd/ra_root.crt
[2026-01-02 07:04:57 UTC] USER=www-data EUID=0 PID=1631777 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-fastorder_admin_gd/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = fastorder_admin_gd
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631792 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631801 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631810 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-fastorder_admin_gd/fastorder_admin_gd.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631819 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-fastorder_admin_gd/fastorder_admin_gd.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631828 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-fastorder_admin_gd/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631837 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631846 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-fastorder_admin_gd/fastorder_admin_gd.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631855 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-fastorder_admin_gd/fastorder_admin_gd_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631864 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631873 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631882 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631891 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631900 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:04:58 UTC] USER=www-data EUID=0 PID=1631909 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1631918 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1631927 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1631936 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1631945 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1631971 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1631980 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1631989 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1631998 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632007 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632016 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632025 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632034 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632043 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632052 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632061 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632071 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632081 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632090 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632099 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632108 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632117 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632126 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632135 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632144 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632153 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:04:59 UTC] USER=www-data EUID=0 PID=1632162 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632171 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632181 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632191 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632200 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632209 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632218 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632227 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632236 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632245 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632254 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632263 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632272 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632281 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632291 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632301 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632310 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632319 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632328 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632337 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632346 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632355 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632364 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632373 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632382 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key.pkcs1
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632391 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd_der.key
[2026-01-02 07:05:00 UTC] USER=www-data EUID=0 PID=1632401 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632411 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632420 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632429 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632439 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632448 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632457 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632466 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632475 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632484 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: fastorder_admin_gd
Node: worker-01
FQDN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/fastorder_admin_gd.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-worker-01.fastorder.com -U fastorder_admin_gd -d postgres
🧱 Connecting via Unix socket to create role and database...
Socket: /var/run/postgresql-identity-sau-main-dev-worker-01:5432
📦 Creating role fastorder_admin_gd...
✅ Role fastorder_admin_gd created
ℹ️ Database fastorder_identity_sau_main_dev_db already exists, skipping creation
[2026-01-02 07:05:01 UTC] USER=www-data EUID=0 PID=1632542 ACTION=passthru ARGS=sudo -u postgres psql -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc
GRANT
✅ Role and DB created via SSL
🔐 Adding user to pg_hba.conf for SSL access...
ℹ️ Using pg_hba.conf: /var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
✅ Added fastorder_admin_gd to pg_hba.conf
🔄 Reloading PostgreSQL configuration...
[2026-01-02 07:05:02 UTC] USER=www-data EUID=0 PID=1632612 ACTION=passthru ARGS=systemctl reload postgresql@identity-sau-main-dev-worker-01.service
✅ PostgreSQL configuration reloaded
🧪 Testing connection for user: fastorder_admin_gd
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[0;34m=== Pre-flight Checks ===[0m
ℹ️ Testing AWS IAM credentials...
✅ AWS IAM credentials are valid
{
"UserId": "AIDAWYLM4MSHFSCGU7QUM",
"Account": "464621692046",
"Arn": "arn:aws:iam::464621692046:user/fo-dev"
}
[0;32m✓[0m AWS Secrets Manager accessible
[0;34m=== Retrieving Credentials from AWS ===[0m
ℹ️ Retrieving PostgreSQL credentials for: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd
ℹ️ Fetching secret: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd
/opt/fastorder/bash/infra_core/cache.sh: line 145: /var/cache/secrets/fastorder_db_identity_sau_main_dev_postgresql_worker-01_fastorder_admin_gd.cache.tmp.1632644: Permission denied
✅ Retrieved from secrets manager: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd
✅ PostgreSQL credentials loaded for worker-01/fastorder_admin_gd: fastorder_admin_gd@db-identity-sau-main-dev-postgresql-worker-01.fastorder.com:5432/fastorder_identity_sau_main_dev_db
[0;32m✓[0m Credentials retrieved: fastorder_admin_gd@db-identity-sau-main-dev-postgresql-worker-01.fastorder.com:5432/fastorder_identity_sau_main_dev_db
[0;34m╔════════════════════════════════════════════╗[0m
[0;34m║ PostgreSQL Test Suite (AWS Secrets MGR) ║[0m
[0;34m╚════════════════════════════════════════════╝[0m
[0;34m=== PostgreSQL Authentication Test ===[0m
[0;31m✗[0m PostgreSQL authentication failed
---- Error Details ----
psql: error: connection to server at "db-identity-sau-main-dev-postgresql-worker-01.fastorder.com" (10.100.1.214), port 5432 failed: root certificate file "/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/fastorder_admin_gd/root.crt" does not exist
Either provide the file, use the system's trusted roots with sslrootcert=system, or change sslmode to disable server certificate verification.
----------------------
❌ User authentication test failed
📋 Password stored securely in AWS Secrets Manager
📋 Secret path: fastorder/db/identity/sau/main/dev/postgresql/worker-01/fastorder_admin_gd
📦 End executing 03-create-role.sh
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[2026-01-02 07:05:11 UTC] USER=www-data EUID=0 PID=1632811 ACTION=fsop ARGS=test -f /var/lib/postgresql/17/identity-sau-main-dev/worker-01/standby.signal
── fast setup ─────────────────────────────────────────────
NAME : identity-sau-main-dev
IDENTIFIER : worker-01
PG HOST : db-identity-sau-main-dev-postgresql-worker-01.fastorder.com:5432
ROLE : debezium_user
DB : fastorder_identity_sau_main_dev_db
SCHEMA : auth
AUTH MODE : scram (scram=password over TLS | cert=mTLS)
SUBNET ALLOW: 10.201.0.0/16
CONNECT /32 : 142.93.238.16
SSL DIR : /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
DNS → 10.100.1.214
CA : /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
🔐 Setting password for user: debezium_user
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
⚠️ ~/.aws/credentials file not found
⚠️ Using environment-based AWS authentication
[0;34m╔════════════════════════════════════════════════════════════╗[0m
[0;34m║ PostgreSQL Password Rotation via AWS Secrets Manager ║[0m
[0;34m╚════════════════════════════════════════════════════════════╝[0m
[0;34mEnvironment Configuration:[0m
[0;34m Service: identity[0m
[0;34m Zone: sau[0m
[0;34m Environment: dev[0m
[0;34m Identifier: worker-01[0m
[0;34mAWS Secret: fastorder/db/identity/sau/main/dev/postgresql/worker-01/debezium_user[0m
[0;34mConnection Info:[0m
[0;34m Socket Dir: /var/run/postgresql-identity-sau-main-dev-worker-01[0m
[0;34m Port: 5432[0m
[0;34mTesting AWS Secrets Manager connectivity...[0m
ℹ️ Testing AWS IAM credentials...
✅ AWS IAM credentials are valid
{
"UserId": "AIDAWYLM4MSHFSCGU7QUM",
"Account": "464621692046",
"Arn": "arn:aws:iam::464621692046:user/fo-dev"
}
[0;34mMethod 1 (PREFERRED): AWS Secrets Manager Rotation[0m
[0;34m────────────────────────────────────────────────────────────[0m
[0;34mThis method uses AWS Secrets Manager's built-in rotation:[0m
[0;34m ✓ Zero-downtime (dual-password window)[0m
[0;34m ✓ Automatic rollback on failure[0m
[0;34m ✓ CloudTrail audit log[0m
[0;34m ✓ CloudWatch metrics[0m
[0;34m ✓ No secret exposure in scripts[0m
[0;34mNon-interactive mode: Proceeding with password rotation automatically[0m
[0;34mGenerating new secure password...[0m
[0;34mUser debezium_user does not exist yet - skipping ALTER, will be created by calling script[0m
[0;32m✓ Password generated for new user: debezium_user[0m
[0;34mStoring password in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/worker-01/debezium_user[0m
ℹ️ Setting PostgreSQL credentials in vault: fastorder/db/identity/sau/main/dev/postgresql/worker-01/debezium_user
ℹ️ Setting secret in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/worker-01/debezium_user
✅ Secret created: fastorder/db/identity/sau/main/dev/postgresql/worker-01/debezium_user
✅ PostgreSQL credentials set in vault: fastorder/db/identity/sau/main/dev/postgresql/worker-01/debezium_user
[0;32m✓ Password stored in AWS Secrets Manager[0m
[0;34mVerifying new credentials...[0m
[0;32m✓ New credentials retrieved from AWS Secrets Manager[0m
[0;34mTesting PostgreSQL connection with new credentials...[0m
[0;32m✓ PostgreSQL connection successful (socket authentication)[0m
[0;32m✓ ╔════════════════════════════════════════════════════════════╗[0m
[0;32m✓ ║ Password Rotation Complete! ║[0m
[0;32m✓ ╚════════════════════════════════════════════════════════════╝[0m
[0;34mSecret: fastorder/db/identity/sau/main/dev/postgresql/worker-01/debezium_user[0m
[0;34mMethod: Direct Update (stored in AWS Secrets Manager)[0m
[0;34mStatus: Completed[0m
[0;34mTo retrieve credentials:[0m
[0;34m # Using Bash library[0m
[0;34m source /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh[0m
[0;34m get_pg_credentials worker-01[0m
[0;34mAudit trail: AWS CloudTrail (for Secrets Manager operations)[0m
[0;32m✓ Done![0m
🔍 Retrieving password from vault with identifier: worker-01/debezium_user
✓ Retrieved password from secrets vault
password : (stored in AWS Secrets Manager)
🔍 TLS chain check...
🔧 Ensuring role and grants…
ℹ️ Role debezium_user exists, updating
[2026-01-02 07:05:27 UTC] USER=www-data EUID=0 PID=1633236 ACTION=passthru ARGS=sudo -u postgres psql -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc
ALTER ROLE
ℹ️ Database fastorder_identity_sau_main_dev_db already exists
[2026-01-02 07:05:27 UTC] USER=www-data EUID=0 PID=1633262 ACTION=passthru ARGS=sudo -u postgres psql -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d fastorder_identity_sau_main_dev_db --no-psqlrc
CREATE SCHEMA
GRANT
GRANT
GRANT
GRANT
ALTER DEFAULT PRIVILEGES
✅ Role/DB/grants ensured.
⚠️ Could not find pg_hba.conf (skipping HBA edits): /var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
🧪 Testing ROLE connection (scram)...
✅ SCRAM+TLS probe OK
🎉 Done.
🔐 Creating replicator role for worker-01...
[WARN] Deadlock prevention library not found: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/role/lib/pg-deadlock-prevention.sh
🔑 Configuring AWS credentials...
✅ Using permanent AWS credentials from /var/www/.aws/credentials
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
── replicator setup ───────────────────────────────────────
NAME : identity-sau-main-dev
IDENTIFIER : worker-01
PG HOST : db-identity-sau-main-dev-postgresql-worker-01.fastorder.com:5432
ROLE : replicator
SSL DIR : /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
DNS → 10.100.1.214
CA : /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
🔍 TLS chain check...
🔧 Ensuring replicator role…
🔐 Checking AWS Secrets Manager for replicator password...
🔑 Generating new secure replicator password...
💾 Storing replicator password in AWS Secrets Manager...
{
"ARN": "arn:aws:secretsmanager:me-central-1:464621692046:secret:fastorder/db/identity/sau/main/dev/postgresql/replicator-plMUla",
"Name": "fastorder/db/identity/sau/main/dev/postgresql/replicator",
"VersionId": "37e2bd76-86f1-48dc-a8f2-2fb93223dab5"
}
✅ Password stored in AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/replicator
ℹ️ Temporarily disabling synchronous_commit to prevent replication deadlock...
NOTICE: Creating role: replicator with password
SET
CREATE ROLE
✅ Replicator role ensured with password authentication.
ℹ️ Password stored in: AWS Secrets Manager
Secret name: fastorder/db/identity/sau/main/dev/postgresql/replicator
🔄 MIGRATION PATH: Password → Certificate Authentication
Current: SCRAM-SHA-256 password auth (production-ready)
Future: Certificate-based auth (requires CA automation)
To migrate: Update pg_hba.conf rules from 'scram-sha-256' to 'cert clientcert=verify-full'
and configure standby to use SSL certificates instead of password
🎉 Done.
✅ Replicator role created for worker-01
[DEBUG] Tracking substep start: steps/01-install/steps/05-setup-service (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 05 setup service...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
ℹ️ Service-specific setup (identity) is handled by parent script
✅ Step 5 completed (service setup delegated to 01-install/run.sh)
🔍 DEBUG_CHECKPOINT_01: Starting service-specific steps discovery
🔍 DEBUG_CHECKPOINT_02: Searching for service folders in: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps
🔍 DEBUG_CHECKPOINT_03: Found directory: destroy
🔍 DEBUG_CHECKPOINT_03: Found directory: iam
🔍 DEBUG_CHECKPOINT_04: Found run.sh in: iam
🔍 DEBUG_CHECKPOINT_03: Found directory: identity
🔍 DEBUG_CHECKPOINT_04: Found run.sh in: identity
🔍 DEBUG_CHECKPOINT_03: Found directory: lib
🔍 DEBUG_CHECKPOINT_03: Found directory: passwords
🔍 DEBUG_CHECKPOINT_03: Found directory: role
🔍 DEBUG_CHECKPOINT_03: Found directory: ssl
🔍 DEBUG_CHECKPOINT_05: Service folders found: iam identity
[INFO] 📚 Detected service folders: iam identity
🔍 DEBUG_CHECKPOINT_06: Preparing to run service: iam at /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/iam/run.sh
[DEBUG] Tracking substep start: steps/01-install/steps/iam (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 🔸 Service: iam
🔍 DEBUG_CHECKPOINT_07: About to execute /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/iam/run.sh with IDENTIFIER=worker-01 IDENTIFIER_PARENT=worker-01
🔍 DEBUG_CHECKPOINT_08: Running iam in AUTO mode
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[0;35m╔════════════════════════════════════════════════════════════════════════════╗[0m
[0;35m║ IAM Database Schema Initialization ║[0m
[0;35m╚════════════════════════════════════════════════════════════════════════════╝[0m
[0;34m[INFO][0m 🟢 Starting IAM schema provisioning...
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: coordinator
[0;34m[INFO][0m VM IP: 142.93.238.16
[0;34m[INFO][0m 📚 Discovered tables: core/01-tenant core/02-realm core/03-identity core/04-device core/05-identity_account core/06-identity_mfa core/07-external_idp_link policy/01-client policy/02-resource policy/03-scope policy/04-permission policy/05-role policy/06-role_permission policy/07-identity_role policy/08-policy_rule policy/09-api_key audit/01-auth_event audit/02-admin_action audit/03-risk_decision audit/04-consent_event
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m Schema: core[0m
[0;34m Core Identity Directory (tenants, realms, identities, devices, MFA)[0m
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m[INFO][0m 🔸 Table [1/20]: core/01-tenant
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.tenant Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Identifier: coordinator
Database: fastorder_identity_sau_main_dev_db
Host: db-identity-sau-main-dev-postgresql.fastorder.com:5432
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔐 Connecting to PostgreSQL over SSL (verify-full + mTLS)...
🗄️ Checking database: fastorder_identity_sau_main_dev_db
ℹ️ Database fastorder_identity_sau_main_dev_db already exists
✅ Connected to database: fastorder_identity_sau_main_dev_db
🔧 Installing extensions...
NOTICE: extension "uuid-ossp" already exists, skipping
CREATE EXTENSION
NOTICE: extension "pgcrypto" already exists, skipping
CREATE EXTENSION
NOTICE: extension "citext" already exists, skipping
CREATE EXTENSION
NOTICE: extension "dblink" already exists, skipping
CREATE EXTENSION
🔧 Installing Citus extension on coordinator...
NOTICE: extension "citus" already exists, skipping
CREATE EXTENSION
✅ Citus extension installed
✅ Extensions installed
🔧 Creating utils schema...
NOTICE: schema "utils" already exists, skipping
CREATE SCHEMA
✅ Utils schema created
🔧 Installing UUIDv7 function...
✅ UUIDv7 function installed
🔧 Creating core schema...
NOTICE: schema "core" already exists, skipping
CREATE SCHEMA
✅ Schema core created
🔧 Creating ENUM types...
DO
✅ ENUM types created
🔧 Creating core.tenant table...
NOTICE: relation "tenant" already exists, skipping
CREATE TABLE
COMMENT
COMMENT
COMMENT
✅ core.tenant created
🔧 Setting up Citus distribution for core.tenant...
✅ Citus distribution configured
🔧 Creating update trigger...
CREATE FUNCTION
ERROR: triggers are not supported on reference tables
ERROR: triggers are not supported on reference tables
✅ Update trigger created
✅ core.tenant initialization complete
[0;32m[OK][0m Table core/01-tenant initialized
[0;34m[INFO][0m 🔸 Table [2/20]: core/02-realm
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.realm Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.realm table...
NOTICE: relation "realm" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_realm_keycloak_id" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_realm_tenant" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ core.realm created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ core.realm initialization complete
[0;32m[OK][0m Table core/02-realm initialized
[0;34m[INFO][0m 🔸 Table [3/20]: core/03-identity
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.identity Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.identity table...
NOTICE: relation "identity" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_identity_unique_email" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_unique_keycloak" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_email" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_keycloak" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_status" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_type" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.identity created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ core.identity initialization complete
[0;32m[OK][0m Table core/03-identity initialized
[0;34m[INFO][0m 🔸 Table [4/20]: core/04-device
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.device Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.device table...
NOTICE: relation "device" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_device_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_device_fingerprint" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_device_trusted" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_device_last_seen" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.device created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ core.device initialization complete
[0;32m[OK][0m Table core/04-device initialized
[0;34m[INFO][0m 🔸 Table [5/20]: core/05-identity_account
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.identity_account Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.identity_account table...
NOTICE: relation "identity_account" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_identity_account_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_account_lockout" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_account_last_login" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.identity_account created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ core.identity_account initialization complete
[0;32m[OK][0m Table core/05-identity_account initialized
[0;34m[INFO][0m 🔸 Table [6/20]: core/06-identity_mfa
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.identity_mfa Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.identity_mfa table...
NOTICE: relation "identity_mfa" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_identity_mfa_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_mfa_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_mfa_active" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.identity_mfa created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ core.identity_mfa initialization complete
[0;32m[OK][0m Table core/06-identity_mfa initialized
[0;34m[INFO][0m 🔸 Table [7/20]: core/07-external_idp_link
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.external_idp_link Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.external_idp_link table...
NOTICE: relation "external_idp_link" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_external_idp_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_external_idp_provider" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_external_idp_email" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.external_idp_link created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ core.external_idp_link initialization complete
[0;32m[OK][0m Table core/07-external_idp_link initialized
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m Schema: policy[0m
[0;34m RBAC/ABAC Authorization (clients, roles, permissions, policies)[0m
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m[INFO][0m 🔸 Table [8/20]: policy/01-client
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.client Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy schema...
NOTICE: schema "policy" already exists, skipping
CREATE SCHEMA
✅ Schema policy created
🔧 Creating ENUM types...
DO
✅ ENUM types created
🔧 Creating policy.client table...
NOTICE: relation "client" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_client_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_client_keycloak" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_client_key" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_client_status" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ policy.client created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
CREATE FUNCTION
DROP TRIGGER
CREATE TRIGGER
✅ policy.client initialization complete
[0;32m[OK][0m Table policy/01-client initialized
[0;34m[INFO][0m 🔸 Table [9/20]: policy/02-resource
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.resource Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.resource table...
NOTICE: relation "resource" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_resource_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_resource_external" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_resource_owner" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.resource created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.resource initialization complete
[0;32m[OK][0m Table policy/02-resource initialized
[0;34m[INFO][0m 🔸 Table [10/20]: policy/03-scope
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.scope Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.scope table...
NOTICE: relation "scope" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_scope_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_scope_name" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.scope created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ policy.scope initialization complete
[0;32m[OK][0m Table policy/03-scope initialized
[0;34m[INFO][0m 🔸 Table [11/20]: policy/04-permission
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.permission Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.permission table...
NOTICE: relation "permission" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_permission_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_permission_name" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_permission_resource" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.permission created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ policy.permission initialization complete
[0;32m[OK][0m Table policy/04-permission initialized
[0;34m[INFO][0m 🔸 Table [12/20]: policy/05-role
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.role Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.role table...
NOTICE: relation "role" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_role_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_role_client" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_role_name" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_role_keycloak" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ policy.role created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ policy.role initialization complete
[0;32m[OK][0m Table policy/05-role initialized
[0;34m[INFO][0m 🔸 Table [13/20]: policy/06-role_permission
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.role_permission Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.role_permission table...
NOTICE: relation "role_permission" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_role_permission_role" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_role_permission_perm" already exists, skipping
CREATE INDEX
COMMENT
✅ policy.role_permission created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.role_permission initialization complete
[0;32m[OK][0m Table policy/06-role_permission initialized
[0;34m[INFO][0m 🔸 Table [14/20]: policy/07-identity_role
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.identity_role Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.identity_role table...
NOTICE: relation "identity_role" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_identity_role_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_role_role" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_role_active" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_role_expires" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.identity_role created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.identity_role initialization complete
[0;32m[OK][0m Table policy/07-identity_role initialized
[0;34m[INFO][0m 🔸 Table [15/20]: policy/08-policy_rule
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.policy_rule Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.policy_rule table...
NOTICE: relation "policy_rule" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_policy_rule_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_policy_rule_enabled" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_policy_rule_priority" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.policy_rule created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ policy.policy_rule initialization complete
[0;32m[OK][0m Table policy/08-policy_rule initialized
[0;34m[INFO][0m 🔸 Table [16/20]: policy/09-api_key
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.api_key Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.api_key table...
NOTICE: relation "api_key" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_api_key_prefix" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_api_key_client" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_api_key_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_api_key_status" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_api_key_expires" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ policy.api_key created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.api_key initialization complete
[0;32m[OK][0m Table policy/09-api_key initialized
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m Schema: audit[0m
[0;34m Audit & Risk Logging (auth events, admin actions, risk decisions)[0m
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m[INFO][0m 🔸 Table [17/20]: audit/01-auth_event
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.auth_event Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit schema...
NOTICE: schema "audit" already exists, skipping
CREATE SCHEMA
✅ Schema audit created
🔧 Creating ENUM types...
DO
✅ ENUM types created
🔧 Creating audit.auth_event table...
NOTICE: relation "auth_event" already exists, skipping
CREATE TABLE
NOTICE: relation "audit.auth_event_2026_01" already exists, skipping
NOTICE: relation "audit.auth_event_2026_02" already exists, skipping
DO
NOTICE: relation "idx_auth_event_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_time" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_result" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_ip" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_session" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_trace" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_risk" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ audit.auth_event created (partitioned)
✅ audit.auth_event initialization complete
[0;32m[OK][0m Table audit/01-auth_event initialized
[0;34m[INFO][0m 🔸 Table [18/20]: audit/02-admin_action
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.admin_action Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit.admin_action table...
NOTICE: relation "admin_action" already exists, skipping
CREATE TABLE
NOTICE: relation "audit.admin_action_2026_01" already exists, skipping
NOTICE: relation "audit.admin_action_2026_02" already exists, skipping
DO
NOTICE: relation "idx_admin_action_actor" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_admin_action_target" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_admin_action_time" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_admin_action_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_admin_action_trace" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ audit.admin_action created (partitioned)
✅ audit.admin_action initialization complete
[0;32m[OK][0m Table audit/02-admin_action initialized
[0;34m[INFO][0m 🔸 Table [19/20]: audit/03-risk_decision
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.risk_decision Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit.risk_decision table...
NOTICE: relation "risk_decision" already exists, skipping
CREATE TABLE
NOTICE: relation "audit.risk_decision_2026_01" already exists, skipping
NOTICE: relation "audit.risk_decision_2026_02" already exists, skipping
DO
NOTICE: relation "idx_risk_decision_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_risk_decision_level" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_risk_decision_decision" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_risk_decision_auth" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_risk_decision_time" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ audit.risk_decision created (partitioned)
✅ audit.risk_decision initialization complete
[0;32m[OK][0m Table audit/03-risk_decision initialized
[0;34m[INFO][0m 🔸 Table [20/20]: audit/04-consent_event
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.consent_event Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit.consent_event table...
NOTICE: relation "consent_event" already exists, skipping
CREATE TABLE
NOTICE: relation "audit.consent_event_2026_01" already exists, skipping
NOTICE: relation "audit.consent_event_2026_02" already exists, skipping
DO
NOTICE: relation "idx_consent_event_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_consent_event_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_consent_event_version" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_consent_event_granted" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_consent_event_time" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ audit.consent_event created (partitioned)
🔧 Creating partition management functions...
CREATE FUNCTION
NOTICE: relation "audit.auth_event_2026_01" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_01
NOTICE: relation "audit.auth_event_2026_02" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_02
NOTICE: relation "audit.auth_event_2026_03" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_03
NOTICE: relation "audit.auth_event_2026_04" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_04
NOTICE: relation "audit.admin_action_2026_01" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_01
NOTICE: relation "audit.admin_action_2026_02" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_02
NOTICE: relation "audit.admin_action_2026_03" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_03
NOTICE: relation "audit.admin_action_2026_04" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_04
NOTICE: relation "audit.risk_decision_2026_01" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_01
NOTICE: relation "audit.risk_decision_2026_02" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_02
NOTICE: relation "audit.risk_decision_2026_03" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_03
NOTICE: relation "audit.risk_decision_2026_04" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_04
NOTICE: relation "audit.consent_event_2026_01" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_01
NOTICE: relation "audit.consent_event_2026_02" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_02
NOTICE: relation "audit.consent_event_2026_03" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_03
NOTICE: relation "audit.consent_event_2026_04" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_04
create_monthly_partitions
---------------------------
(1 row)
CREATE VIEW
CREATE FUNCTION
COMMENT
COMMENT
✅ Partition management functions created
✅ audit.consent_event initialization complete
[0;32m[OK][0m Table audit/04-consent_event initialized
[0;35m════════════════════════════════════════════════════════════════════════════[0m
[0;32m[OK][0m ✅ IAM Schema Initialization Complete!
[0;32m[OK][0m All 20 tables initialized successfully
[0;34mSchemas created:[0m
• core - Identity directory (tenant, realm, identity, devices, MFA)
• policy - Authorization (clients, roles, permissions, policies, API keys)
• audit - Logging (auth events, admin actions, risk decisions, consent)
[0;34mDesign highlights:[0m
• Citus-ready with tenant_id distribution key
• NIST 800-63 identity compliance
• PCI DSS 4.0 audit logging
• GDPR consent tracking
• Keycloak integration via ID references
[0;35m════════════════════════════════════════════════════════════════════════════[0m
🔍 DEBUG_CHECKPOINT_06: Preparing to run service: identity at /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/run.sh
[DEBUG] Tracking substep start: steps/01-install/steps/identity (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 🔸 Service: identity
🔍 DEBUG_CHECKPOINT_07: About to execute /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/run.sh with IDENTIFIER=worker-01 IDENTIFIER_PARENT=worker-01
🔍 DEBUG_CHECKPOINT_08: Running identity in AUTO mode
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] 🟢 Starting PostgreSQL provisioning for identity in sau-dev...
[INFO] Environment: identity-sau-main-dev
[INFO] Identifier: coordinator
[INFO] VM IP: 142.93.238.16
🔍 DEBUG_CHECKPOINT_A1: identity/run.sh started for SERVICE=identity
🔍 DEBUG_CHECKPOINT_A2: Checking SERVICE_ROOT: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity
🔍 DEBUG_CHECKPOINT_A3: SERVICE_ROOT exists, discovering table folders
🔍 DEBUG_CHECKPOINT_A4: Found subfolder: auth
🔍 DEBUG_CHECKPOINT_A4b: Checking for nested schema layout in: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth
🔍 DEBUG_CHECKPOINT_A4c: Found nested steps dir: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps (display: auth/login)
🔍 DEBUG_CHECKPOINT_A5: Table step dirs discovered: auth/login|/opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps
🔍 DEBUG_CHECKPOINT_A6: Checking if we have table folders to process
[INFO] 📚 Detected grouped table folders under identity/: auth/login
🔍 DEBUG_CHECKPOINT_A7: Current IDENTIFIER=coordinator
🔍 DEBUG_CHECKPOINT_A8_PROCEED: Processing tables on coordinator/main node
🔍 DEBUG_CHECKPOINT_A9: Processing table: auth/login at /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps
[INFO] 🔸 Table group: auth/login
🔍 DEBUG_CHECKPOINT_A10: About to run numbered steps for table: auth/login
🔍 DEBUG_CHECKPOINT_B1: run_all_numbered_steps_in_dir called for dir=/opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps table=auth/login
🔍 DEBUG_CHECKPOINT_B2: Found 1 numbered steps: 01-init-schema.sh
🔍 DEBUG_CHECKPOINT_B3: About to run step: 01-init-schema.sh
Ab substep 0 compelete start
[DEBUG] Tracking substep start: steps/01-install/steps/identity/auth/login/01-init-schema (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
Ab substep 0 compelete start
[INFO] 📦 01 init schema...
Ab substep 1 compelete start
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing auth.login_account table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Identifier: coordinator
Database: fastorder_identity_sau_main_dev_db
Host: db-identity-sau-main-dev-postgresql.fastorder.com:5432
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔐 Connecting to PostgreSQL over SSL (verify-full + mTLS)...
🗄️ Checking database: fastorder_identity_sau_main_dev_db
ℹ️ Database fastorder_identity_sau_main_dev_db already exists
✅ Connected to database: fastorder_identity_sau_main_dev_db
ℹ️ Checking synchronous replication configuration...
synchronous_standby_names: ''
Connected standbys: 0
ℹ️ Synchronous replication not configured (standbys will be added later)
🔧 Installing extensions...
NOTICE: extension "uuid-ossp" already exists, skipping
CREATE EXTENSION
NOTICE: extension "dblink" already exists, skipping
CREATE EXTENSION
🔧 Installing Citus extension on coordinator...
NOTICE: extension "citus" already exists, skipping
CREATE EXTENSION
✅ Citus extension installed
✅ Extensions installed
🔧 Installing UUIDv7 function...
✅ UUIDv7 function installed
🔧 Creating auth schema...
NOTICE: schema "auth" already exists, skipping
CREATE SCHEMA
✅ Schema created
🔧 Creating account_status ENUM...
DO
✅ ENUM created
🔧 Creating auth.login_account table...
NOTICE: relation "login_account" already exists, skipping
CREATE TABLE
✅ Table created (Citus-compatible with region_hint in all constraints)
🔧 Creating indexes...
NOTICE: relation "idx_login_account_email" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_login_account_username" already exists, skipping
CREATE INDEX
✅ Indexes created
ℹ️ Table already registered with Citus
🎉 Schema initialization complete for fastorder_identity_sau_main_dev_db
ℹ️ Skipping LISTEN/NOTIFY trigger on coordinator
CDC via Debezium is the primary change tracking mechanism
📊 Registering environment in monitoring database (obs schema)...
Topology: /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
Resource IP: 142.93.238.16
⚠️ Could not connect to monitoring database, skipping registration
You can manually register later using:
/opt/fastorder/bash/scripts/env_app_setup/setup/04-postgresql/steps/register-authN-af-aaaa1-dev.sh
==========================================
✅ Schema initialization complete!
==========================================
Ab substep 1 compelete end
Ab substep 2 compelete start
Ab substep 2 compelete end
🔍 DEBUG_CHECKPOINT_B4: Completed step: 01-init-schema.sh
🔍 DEBUG_CHECKPOINT_B5: All numbered steps completed for /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps
🔍 DEBUG_CHECKPOINT_A11: Completed numbered steps for table: auth/login
compeleted here
🔍 DEBUG_CHECKPOINT_A12: All tables processed
End of 04-postgresql/steps/01-install/steps/identity/run.sh
[0;32m✓[0m ✅ Worker worker-01 setup completed
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Setting up standby replicas (1 per worker)…
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
→ Setting up standby: worker-01-standby-01 (replica of worker-01)
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] 📁 Initializing log directories...
[2026-01-02 07:07:15 UTC] USER=unknown EUID=33 PID=1636650 ACTION=fsop ARGS=mkdir -p /var/log/fastorder/metrics
[2026-01-02 07:07:15 UTC] USER=unknown EUID=33 PID=1636657 ACTION=fsop ARGS=mkdir -p /var/log/fastorder/audit
[2026-01-02 07:07:15 UTC] USER=unknown EUID=33 PID=1636664 ACTION=fsop ARGS=mkdir -p /var/log/fastorder/provisioning
[2026-01-02 07:07:15 UTC] USER=unknown EUID=33 PID=1636671 ACTION=fsop ARGS=chmod 777 /var/log/fastorder/metrics
[2026-01-02 07:07:15 UTC] USER=unknown EUID=33 PID=1636678 ACTION=fsop ARGS=chmod 777 /var/log/fastorder/audit
[2026-01-02 07:07:15 UTC] USER=unknown EUID=33 PID=1636685 ACTION=fsop ARGS=chmod 777 /var/log/fastorder/provisioning
/opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/run.sh: line 41: ok: command not found
[INFO] 🟢 Starting PostgreSQL provisioning for identity in sau-dev...
[INFO] Environment: identity-sau-main-dev
[INFO] Identifier: worker-01-standby-01
[INFO] VM IP: 142.93.238.16
[DEBUG] RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3 JOB_UUID=438ff2f1-77b4-4b8d-aa09-3d1698934d76
[DEBUG] Tracking substep start: steps/01-install/steps/00-configure-network-hosts (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 00 configure network hosts...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[WARN] Could not find PostgreSQL IP for worker-01-standby-01 in topology.json, allocating new VM IP...
/opt/fastorder/bash/scripts/env_app_setup/lib/env-orchestrator/lib/common.sh: line 261: echo: write error: Broken pipe
[INFO] Allocated new VM IP: 10.100.1.211 for db-worker-01-standby-01-postgresql
[INFO] ═══════════════════════════════════════════════════════════════
[INFO] CONFIGURING POSTGRESQL NETWORK & DNS
[INFO] ═══════════════════════════════════════════════════════════════
[INFO] Environment: identity-sau-main-dev
[INFO] Identifier: worker-01-standby-01
[INFO] PostgreSQL IP: 10.100.1.211
[INFO] Primary hostname: db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com
[INFO] Adding /etc/hosts entry for worker-01-standby-01...
[INFO] db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com → 10.100.1.211
[INFO] ➕ Adding db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com → 10.100.1.211
[0;32m✅[0m ✅ Added: db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com → 10.100.1.211
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;32m✅[0m ✅ Network & DNS configuration complete
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[INFO] Verifying /etc/hosts entries:
10.100.1.211 db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com
[DEBUG] Tracking substep start: steps/01-install/steps/01-prepare-ssl-server-postgres (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 01 prepare ssl server postgres...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
📦 PostgreSQL Server Certificate Generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau (Saudi Arabia)
Branch: main
Env: dev
Node: worker-01-standby-01
Primary CN: identity-sau-main-dev.fastorder.com
Alt CN: identity-sau-main-dev.fastorder.com
VM IP: 142.93.238.16
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ Removing existing server certificates (preserving client certs)...
[2026-01-02 07:07:20 UTC] USER=www-data EUID=0 PID=1637316 ACTION=fsop ARGS=rm -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
✅ Ensuring directories exist: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01 and /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:20 UTC] USER=www-data EUID=0 PID=1637325 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
🔐 Generating 4096-bit private key...
[2026-01-02 07:07:20 UTC] USER=www-data EUID=0 PID=1637335 ACTION=fsop ARGS=chmod 755 /tmp/pg-cert-gen-1637279
[2026-01-02 07:07:20 UTC] USER=www-data EUID=0 PID=1637344 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-cert-gen-1637279/ra_root.crt
[2026-01-02 07:07:20 UTC] USER=www-data EUID=0 PID=1637353 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-cert-gen-1637279/ra_root.key
[2026-01-02 07:07:21 UTC] USER=www-data EUID=0 PID=1637362 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1637279/ra_root.crt
[2026-01-02 07:07:21 UTC] USER=www-data EUID=0 PID=1637371 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1637279/ra_root.key
📝 Creating certificate signing request (CSR)...
📜 Signing certificate with internal CA...
Certificate request self-signature ok
subject=C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = identity-sau-main-dev.fastorder.com
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637410 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1637279/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637419 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1637279/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637428 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
📋 Setting up CA certificate...
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637437 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1637279/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637446 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637455 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637464 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Using CA certificate: /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt
🚚 Setting up key in private directory...
Key already in correct location (CERT_DIR == KEY_DIR)
🔒 Securing key and cert permissions...
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637475 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637484 ACTION=fsop ARGS=chmod 600 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637493 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637502 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637511 ACTION=fsop ARGS=chown root:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:26 UTC] USER=www-data EUID=0 PID=1637520 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
🔍 Verifying certificate...
Certificate details:
Subject: C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = identity-sau-main-dev.fastorder.com
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
--
X509v3 Subject Alternative Name:
DNS:identity-sau-main-dev.fastorder.com, DNS:identity-sau-main-dev.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-worker-01-standby-01, DNS:localhost, DNS:db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com, IP Address:142.93.238.16, IP Address:127.0.0.1
X509v3 Subject Key Identifier:
⚠️ Certificate chain verification: FAILED (but certificate may still work)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ PostgreSQL Server Certificate Generated Successfully!
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Node: worker-01-standby-01
Primary CN: identity-sau-main-dev.fastorder.com
Certificate files installed:
📜 Server cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
🔑 Server key: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
🏛️ CA cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt (ca.crt symlink also available)
To use these certificates in PostgreSQL:
1. Update postgresql.conf:
ssl = on
ssl_cert_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt'
ssl_key_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key'
ssl_ca_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt'
2. Restart PostgreSQL:
command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh passthru systemctl restart postgresql@identity-sau-main-dev-worker-01-standby-01.service
3. Test SSL connection:
psql "host=identity-sau-main-dev.fastorder.com port=5432 user=postgres sslmode=verify-full"
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: postgres
Identifier: worker-01-standby-01
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: worker-01-standby-01
User (CN): postgres
Hostname: db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:07:27 UTC] USER=www-data EUID=0 PID=1637574 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-worker-01-standby-01-postgres
[2026-01-02 07:07:27 UTC] USER=www-data EUID=0 PID=1637583 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-worker-01-standby-01-postgres/ra_root.crt
[2026-01-02 07:07:27 UTC] USER=www-data EUID=0 PID=1637592 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-worker-01-standby-01-postgres/ra_root.key
[2026-01-02 07:07:27 UTC] USER=www-data EUID=0 PID=1637601 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-standby-01-postgres/ra_root.crt
[2026-01-02 07:07:27 UTC] USER=www-data EUID=0 PID=1637610 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-standby-01-postgres/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = postgres
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:27 UTC] USER=www-data EUID=0 PID=1637626 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:27 UTC] USER=www-data EUID=0 PID=1637635 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:27 UTC] USER=www-data EUID=0 PID=1637644 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/postgres.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637653 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637662 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637671 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637680 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/postgres.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637689 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/postgres_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637698 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637707 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637716 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637725 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637734 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637743 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637752 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637761 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637787 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637796 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637805 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637814 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637823 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637832 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637841 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637850 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637860 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637869 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637878 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:28 UTC] USER=www-data EUID=0 PID=1637888 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637898 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637907 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637916 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637925 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637934 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637943 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637952 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637961 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637970 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637979 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637988 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1637998 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638008 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638017 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638026 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638035 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638044 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638053 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638062 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638071 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638080 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638089 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638098 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638108 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638118 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638127 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638136 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:29 UTC] USER=www-data EUID=0 PID=1638145 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638156 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638166 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638176 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638185 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638194 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638203 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638212 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638222 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638232 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638241 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638250 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638259 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638268 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638277 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638287 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638297 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:07:30 UTC] USER=www-data EUID=0 PID=1638306 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: postgres
Node: worker-01-standby-01
FQDN: db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com -U postgres -d postgres
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: postgres
Identifier: worker-01-standby-01
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: worker-01-standby-01
User (CN): postgres
Hostname: db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:07:31 UTC] USER=www-data EUID=0 PID=1638347 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-worker-01-standby-01-postgres
[2026-01-02 07:07:31 UTC] USER=www-data EUID=0 PID=1638356 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-worker-01-standby-01-postgres/ra_root.crt
[2026-01-02 07:07:31 UTC] USER=www-data EUID=0 PID=1638365 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-worker-01-standby-01-postgres/ra_root.key
[2026-01-02 07:07:31 UTC] USER=www-data EUID=0 PID=1638374 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-standby-01-postgres/ra_root.crt
[2026-01-02 07:07:31 UTC] USER=www-data EUID=0 PID=1638383 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-standby-01-postgres/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = postgres
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638398 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638407 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638416 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/postgres.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638425 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638434 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638443 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638453 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/postgres.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638464 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-standby-01-postgres/postgres_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638473 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638482 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638491 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638500 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638509 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638518 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638527 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638536 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:32 UTC] USER=www-data EUID=0 PID=1638545 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638554 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638580 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638589 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638598 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638607 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638616 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638625 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638634 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638643 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638652 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638661 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638670 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638680 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638690 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638699 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638708 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638717 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638726 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638735 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638744 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638753 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638762 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:33 UTC] USER=www-data EUID=0 PID=1638771 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638780 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638790 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638800 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638809 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638821 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638838 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638851 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638865 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638876 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638885 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638894 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638903 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638912 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638923 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638933 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638942 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638952 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638961 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638970 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638979 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638988 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1638997 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1639006 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1639015 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key.pkcs1
[2026-01-02 07:07:34 UTC] USER=www-data EUID=0 PID=1639024 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/postgres_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres_der.key
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639034 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639044 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639053 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639062 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639071 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639080 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639089 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639098 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639107 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:07:35 UTC] USER=www-data EUID=0 PID=1639116 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: postgres
Node: worker-01-standby-01
FQDN: db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com -U postgres -d postgres
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: replicator
Identifier: worker-01
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: worker-01
User (CN): replicator
Hostname: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639157 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-worker-01-replicator
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639166 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-worker-01-replicator/ra_root.crt
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639175 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-worker-01-replicator/ra_root.key
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639184 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-replicator/ra_root.crt
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639193 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-replicator/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = replicator
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639207 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639216 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639225 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639234 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639243 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639252 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639261 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639270 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639279 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639288 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639297 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639306 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639315 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639324 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:07:36 UTC] USER=www-data EUID=0 PID=1639333 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639342 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639351 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639360 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639386 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639395 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639404 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639413 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639422 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639431 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639440 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639449 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639458 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639467 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639476 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639486 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639498 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639507 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639516 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639525 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639534 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639543 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639552 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639561 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639570 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:07:37 UTC] USER=www-data EUID=0 PID=1639579 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639588 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639598 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639608 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639617 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639630 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639647 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639674 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639702 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639734 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639762 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639778 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639787 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639797 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639807 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639817 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639826 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639835 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639844 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639853 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639862 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639871 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639880 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639889 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639898 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639907 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:07:38 UTC] USER=www-data EUID=0 PID=1639917 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:39 UTC] USER=www-data EUID=0 PID=1639927 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:39 UTC] USER=www-data EUID=0 PID=1639936 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:39 UTC] USER=www-data EUID=0 PID=1639945 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:07:39 UTC] USER=www-data EUID=0 PID=1639954 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:07:39 UTC] USER=www-data EUID=0 PID=1639963 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:07:39 UTC] USER=www-data EUID=0 PID=1639972 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:07:39 UTC] USER=www-data EUID=0 PID=1639981 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:07:39 UTC] USER=www-data EUID=0 PID=1639990 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:07:39 UTC] USER=www-data EUID=0 PID=1639999 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: replicator
Node: worker-01
FQDN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-worker-01.fastorder.com -U replicator -d postgres
[DEBUG] Tracking substep start: steps/01-install/steps/02-setup-pg-instance (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 02 setup pg instance...
[0;34m[DEADLOCK-PREVENTION][0m Deadlock prevention library loaded
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
🔑 Configuring AWS credentials...
✅ Using permanent AWS credentials from /var/www/.aws/credentials
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔑 Configuring AWS credentials...
[WARN] ~/.aws/credentials file not found
[WARN] AWS operations may require SSO login
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[INFO] Using existing db-worker-01-standby-01-postgresql environment: db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com (10.100.1.211)
[INFO] PostgreSQL will listen on application-specific IP: 10.100.1.211
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: worker-01-standby-01
[0;34m[INFO][0m Data dir: /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01
[0;34m[INFO][0m Port: 5432
[0;34m[INFO][0m Hostname: db-identity-sau-main-dev-postgresql-worker-01-standby-01
[2026-01-02 07:07:41 UTC] USER=www-data EUID=0 PID=1640094 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:41 UTC] USER=www-data EUID=0 PID=1640115 ACTION=fsop ARGS=chmod 755 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:41 UTC] USER=www-data EUID=0 PID=1640136 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:41 UTC] USER=www-data EUID=0 PID=1640157 ACTION=fsop ARGS=chown root:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[1;33m[WARN][0m Server certificate not found at /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
[0;34m[INFO][0m Generating server certificate using ssl/server.sh...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
📦 PostgreSQL Server Certificate Generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau (Saudi Arabia)
Branch: main
Env: dev
Node: worker-01-standby-01
Primary CN: identity-sau-main-dev.fastorder.com
Alt CN: identity-sau-main-dev.fastorder.com
VM IP: 142.93.238.16
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ Removing existing server certificates (preserving client certs)...
[2026-01-02 07:07:42 UTC] USER=www-data EUID=0 PID=1640198 ACTION=fsop ARGS=rm -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
✅ Ensuring directories exist: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01 and /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:42 UTC] USER=www-data EUID=0 PID=1640207 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
🔐 Generating 4096-bit private key...
[2026-01-02 07:07:42 UTC] USER=www-data EUID=0 PID=1640217 ACTION=fsop ARGS=chmod 755 /tmp/pg-cert-gen-1640164
[2026-01-02 07:07:42 UTC] USER=www-data EUID=0 PID=1640226 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-cert-gen-1640164/ra_root.crt
[2026-01-02 07:07:42 UTC] USER=www-data EUID=0 PID=1640235 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-cert-gen-1640164/ra_root.key
[2026-01-02 07:07:42 UTC] USER=www-data EUID=0 PID=1640244 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1640164/ra_root.crt
[2026-01-02 07:07:42 UTC] USER=www-data EUID=0 PID=1640253 ACTION=fsop ARGS=chmod 644 /tmp/pg-cert-gen-1640164/ra_root.key
📝 Creating certificate signing request (CSR)...
📜 Signing certificate with internal CA...
Certificate request self-signature ok
subject=C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = identity-sau-main-dev.fastorder.com
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640291 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1640164/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640300 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1640164/server.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640309 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
📋 Setting up CA certificate...
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640318 ACTION=fsop ARGS=cp /tmp/pg-cert-gen-1640164/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640327 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640336 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640345 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt
✅ Using CA certificate: /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt
🚚 Setting up key in private directory...
Key already in correct location (CERT_DIR == KEY_DIR)
🔒 Securing key and cert permissions...
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640356 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640365 ACTION=fsop ARGS=chmod 600 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640374 ACTION=fsop ARGS=chown postgres:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640383 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640393 ACTION=fsop ARGS=chown root:postgres /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640402 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01
🔍 Verifying certificate...
Certificate details:
Subject: C = SA, ST = Riyadh, L = Riyadh, O = FastOrder, OU = PostgreSQL, CN = identity-sau-main-dev.fastorder.com
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
--
X509v3 Subject Alternative Name:
DNS:identity-sau-main-dev.fastorder.com, DNS:identity-sau-main-dev.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com, DNS:db-identity-sau-main-dev-postgresql-worker-01-standby-01, DNS:localhost, DNS:db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com, IP Address:142.93.238.16, IP Address:127.0.0.1
X509v3 Subject Key Identifier:
⚠️ Certificate chain verification: FAILED (but certificate may still work)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
✅ PostgreSQL Server Certificate Generated Successfully!
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Node: worker-01-standby-01
Primary CN: identity-sau-main-dev.fastorder.com
Certificate files installed:
📜 Server cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
🔑 Server key: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
🏛️ CA cert: /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt (ca.crt symlink also available)
To use these certificates in PostgreSQL:
1. Update postgresql.conf:
ssl = on
ssl_cert_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt'
ssl_key_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key'
ssl_ca_file = '/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/root.crt'
2. Restart PostgreSQL:
command sudo -n /usr/local/bin/fastorder-provisioning-wrapper.sh passthru systemctl restart postgresql@identity-sau-main-dev-worker-01-standby-01.service
3. Test SSL connection:
psql "host=identity-sau-main-dev.fastorder.com port=5432 user=postgres sslmode=verify-full"
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m ✅ Using canonical certificate path (hardened, ProtectHome=true compatible)
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640431 ACTION=fsop ARGS=test -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.crt
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640440 ACTION=fsop ARGS=test -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/server.key
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640449 ACTION=fsop ARGS=test -f /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt
[0;32m[OK][0m mTLS certificates OK (server cert + client certs verified) and keys secured
[0;34m[INFO][0m Preflight: stopping any conflicting Postgres services/processes on port 5432…
[2026-01-02 07:07:46 UTC] USER=www-data EUID=0 PID=1640470 ACTION=passthru ARGS=systemctl stop postgresql@identity-sau-main-dev-worker-01-standby-01.service
[2026-01-02 07:07:47 UTC] USER=www-data EUID=0 PID=1640491 ACTION=passthru ARGS=systemctl stop postgresql
[0;32m[OK][0m No conflicting Postgres left on port 5432
[0;32m[OK][0m Generated new postgres password for initdb
[2026-01-02 07:08:13 UTC] USER=www-data EUID=0 PID=1640740 ACTION=fsop ARGS=chown postgres:postgres /tmp/.pg_pwfile.4fJ0SB
[2026-01-02 07:08:14 UTC] USER=www-data EUID=0 PID=1640761 ACTION=fsop ARGS=chmod 600 /tmp/.pg_pwfile.4fJ0SB
[2026-01-02 07:08:14 UTC] USER=www-data EUID=0 PID=1640783 ACTION=fsop ARGS=mkdir -p /var/lib/postgresql/17/identity-sau-main-dev
[2026-01-02 07:08:14 UTC] USER=www-data EUID=0 PID=1640805 ACTION=fsop ARGS=chown postgres:postgres /var/lib/postgresql/17/identity-sau-main-dev
[2026-01-02 07:08:14 UTC] USER=www-data EUID=0 PID=1640827 ACTION=fsop ARGS=chmod 755 /var/lib/postgresql/17/identity-sau-main-dev
[0;34m[INFO][0m This is a standby. Using pg_basebackup from primary (worker-01)...
[0;34m[INFO][0m Setting up replicator role and slot on primary (worker-01)...
ℹ️ Scanning primary for stuck queries from previous failed attempts...
ℹ️ Scanning for stuck queries (timeout: 30s)...
ℹ️ No stuck queries found
[WARN] Deadlock prevention library not found: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/role/lib/pg-deadlock-prevention.sh
🔑 Configuring AWS credentials...
✅ Using permanent AWS credentials from /var/www/.aws/credentials
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
── replicator setup ───────────────────────────────────────
NAME : identity-sau-main-dev
IDENTIFIER : worker-01
PG HOST : db-identity-sau-main-dev-postgresql-worker-01.fastorder.com:5432
ROLE : replicator
SLOT : worker_01_standby_01
SSL DIR : /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
DNS → 10.100.1.214
CA : /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
🔍 TLS chain check...
🔧 Ensuring replicator role…
🔐 Checking AWS Secrets Manager for replicator password...
✅ Retrieved replicator password from AWS Secrets Manager
ℹ️ Temporarily disabling synchronous_commit to prevent replication deadlock...
NOTICE: Role replicator already exists, updating password and ensuring REPLICATION privilege
SET
ALTER ROLE
✅ Replicator role ensured with password authentication.
ℹ️ Password stored in: AWS Secrets Manager
Secret name: fastorder/db/identity/sau/main/dev/postgresql/replicator
🔄 MIGRATION PATH: Password → Certificate Authentication
Current: SCRAM-SHA-256 password auth (production-ready)
Future: Certificate-based auth (requires CA automation)
To migrate: Update pg_hba.conf rules from 'scram-sha-256' to 'cert clientcert=verify-full'
and configure standby to use SSL certificates instead of password
🔧 Ensuring replication slot: worker_01_standby_01…
🆕 Creating replication slot worker_01_standby_01
SET
pg_create_physical_replication_slot
-------------------------------------
(worker_01_standby_01,)
(1 row)
✅ Replication slot worker_01_standby_01 created.
🎉 Done.
[0;32m[OK][0m Replicator role and slot created on primary
[0;34m[INFO][0m Creating replicator client certificates for connecting to primary (worker-01)...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: replicator
Identifier: worker-01
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: worker-01
User (CN): replicator
Hostname: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:08:19 UTC] USER=www-data EUID=0 PID=1641074 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-worker-01-replicator
[2026-01-02 07:08:19 UTC] USER=www-data EUID=0 PID=1641083 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-worker-01-replicator/ra_root.crt
[2026-01-02 07:08:19 UTC] USER=www-data EUID=0 PID=1641092 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-worker-01-replicator/ra_root.key
[2026-01-02 07:08:19 UTC] USER=www-data EUID=0 PID=1641101 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-replicator/ra_root.crt
[2026-01-02 07:08:19 UTC] USER=www-data EUID=0 PID=1641112 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-worker-01-replicator/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = replicator
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:08:19 UTC] USER=www-data EUID=0 PID=1641133 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641142 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641151 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641160 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641169 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641178 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641187 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641196 ACTION=fsop ARGS=cp -f /tmp/pg-client-worker-01-replicator/replicator_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641205 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641214 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641223 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641232 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641241 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641250 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641259 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641268 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641277 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641286 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641312 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641321 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641330 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641339 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641348 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641357 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641366 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:08:20 UTC] USER=www-data EUID=0 PID=1641375 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641384 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641393 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641402 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641412 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641422 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641431 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641440 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641449 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641458 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641467 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641476 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641485 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641494 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641503 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641512 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641522 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641533 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641542 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641551 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641560 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641569 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641578 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641587 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641596 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641605 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:08:21 UTC] USER=www-data EUID=0 PID=1641616 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641625 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641635 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641645 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641654 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641663 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641672 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641681 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641690 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641700 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641709 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641718 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641727 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641736 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator_der.key
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641746 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01 → /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641756 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641765 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641776 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641785 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641794 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641803 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641812 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641821 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:08:22 UTC] USER=www-data EUID=0 PID=1641830 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: replicator
Node: worker-01
FQDN: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-worker-01.fastorder.com -U replicator -d postgres
[0;32m[OK][0m Replicator certificate created for worker-01 in /home/postgres/
[0;34m[INFO][0m Using replicator certificates from primary worker-01...
[2026-01-02 07:08:23 UTC] USER=www-data EUID=0 PID=1641862 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key.pkcs1
[2026-01-02 07:08:23 UTC] USER=www-data EUID=0 PID=1641883 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.key
[2026-01-02 07:08:23 UTC] USER=www-data EUID=0 PID=1641904 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/replicator.crt
[0;32m[OK][0m Replicator certificates verified at /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[0;32m[OK][0m root.crt verified at /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01
[0;34m[INFO][0m Updating primary pg_hba.conf to allow replication...
[0;34m[INFO][0m Standby IP: 10.100.1.211/32 (standby's source IP)
[0;34m[INFO][0m Primary application IP: 10.100.1.214/32 (for local pg_basebackup)
[0;34m[INFO][0m Primary DNS IP: 10.100.1.214/32 (DNS resolution of db-identity-sau-main-dev-postgresql-worker-01.fastorder.com)
[2026-01-02 07:08:23 UTC] USER=www-data EUID=0 PID=1641934 ACTION=passthru ARGS=grep -qxF # BEGIN standby-replication (managed) /var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
[2026-01-02 07:08:23 UTC] USER=www-data EUID=0 PID=1641978 ACTION=passthru ARGS=awk -v begin=# BEGIN standby-replication (managed) -v end=# END standby-replication (managed) -v rule=hostssl replication replicator 10.100.1.211/32 scram-sha-256
$0==begin {inside=1}
inside && $0==rule {found=1}
$0==end {inside=0}
END {exit found?0:1}
/var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
[2026-01-02 07:08:24 UTC] USER=www-data EUID=0 PID=1642002 ACTION=passthru ARGS=sed -i /^# END standby-replication (managed)$/i hostssl replication replicator 10.100.1.211/32 scram-sha-256 /var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
[2026-01-02 07:08:24 UTC] USER=www-data EUID=0 PID=1642023 ACTION=passthru ARGS=awk -v begin=# BEGIN standby-replication (managed) -v end=# END standby-replication (managed) -v rule=hostssl replication replicator 10.100.1.214/32 scram-sha-256
$0==begin {inside=1}
inside && $0==rule {found=1}
$0==end {inside=0}
END {exit found?0:1}
/var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
[2026-01-02 07:08:24 UTC] USER=www-data EUID=0 PID=1642047 ACTION=passthru ARGS=sed -i /^# END standby-replication (managed)$/i hostssl replication replicator 10.100.1.214/32 scram-sha-256 /var/lib/postgresql/17/identity-sau-main-dev/worker-01/pg_hba.conf
[0;34m[INFO][0m Reloading primary PostgreSQL service...
[2026-01-02 07:08:24 UTC] USER=www-data EUID=0 PID=1642068 ACTION=passthru ARGS=systemctl reload postgresql@identity-sau-main-dev-worker-01.service
[0;32m[OK][0m Primary pg_hba.conf updated and service reloaded
[0;34m[INFO][0m Primary host: db-identity-sau-main-dev-postgresql-worker-01.fastorder.com
[0;34m[INFO][0m Using replicator cert: /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt
[0;34m[INFO][0m Using replicator key: /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key (PKCS#8 format)
[0;34m[INFO][0m Using CA cert: /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[0;34m[INFO][0m Verifying postgres user can access certificates...
[0;31m[ERR][0m postgres user CANNOT read /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[0;34m[INFO][0m File permissions:
lrwxrwxrwx 1 postgres ssl-cert 72 Jan 2 07:08 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt -> /etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01/root.crt
[0;34m[INFO][0m Parent directory permissions:
drwx------ 2 postgres postgres 4096 Jan 2 07:08 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
drwx------ 5 postgres postgres 4096 Jan 2 07:07 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[1;33m[WARN][0m Attempting to fix permissions (/usr/local/bin/fastorder-provisioning-wrapper.sh required)...
[0;34m[INFO][0m Fixing /home/postgres/ directory...
[2026-01-02 07:08:24 UTC] USER=www-data EUID=0 PID=1642136 ACTION=fsop ARGS=chmod 755 /home/postgres/
[0;34m[INFO][0m Fixing /home/postgres/ssl/.postgresql/...
[2026-01-02 07:08:25 UTC] USER=www-data EUID=0 PID=1642157 ACTION=fsop ARGS=chmod 755 /home/postgres/ssl/.postgresql/
[0;34m[INFO][0m Fixing parent directory: /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:08:25 UTC] USER=www-data EUID=0 PID=1642180 ACTION=fsop ARGS=chmod 755 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[0;34m[INFO][0m Fixing certificate directory: /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[2026-01-02 07:08:25 UTC] USER=www-data EUID=0 PID=1642201 ACTION=fsop ARGS=chmod 755 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01
[0;34m[INFO][0m Fixing CA certificate: /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[2026-01-02 07:08:25 UTC] USER=www-data EUID=0 PID=1642222 ACTION=fsop ARGS=chmod 644 /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt
[0;32m[OK][0m Permissions fixed
[0;32m[OK][0m postgres user can now read /home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt after permission fix
[2026-01-02 07:08:25 UTC] USER=www-data EUID=0 PID=1642243 ACTION=fsop ARGS=mkdir -p /var/run/postgresql-identity-sau-main-dev-worker-01-standby-01
[2026-01-02 07:08:25 UTC] USER=www-data EUID=0 PID=1642264 ACTION=fsop ARGS=chown postgres:postgres /var/run/postgresql-identity-sau-main-dev-worker-01-standby-01
[2026-01-02 07:08:25 UTC] USER=www-data EUID=0 PID=1642285 ACTION=fsop ARGS=chmod 2775 /var/run/postgresql-identity-sau-main-dev-worker-01-standby-01
[0;34m[INFO][0m Checking primary database size before pg_basebackup...
[0;34m[INFO][0m Total primary database size: 29 MB
[0;34m[INFO][0m Estimated transfer time: ~0 minutes (at 10MB/s with compression)
[0;34m[INFO][0m Retrieving replicator password from AWS Secrets Manager: fastorder/db/identity/sau/main/dev/postgresql/replicator
[0;32m[OK][0m Replicator password retrieved successfully
[0;34m[INFO][0m Starting pg_basebackup...
[2026-01-02 07:08:28 UTC] USER=www-data EUID=0 PID=1642358 ACTION=passthru ARGS=sudo -u postgres env PGPASSWORD=qrzga0rZrBWHXjHNfE1t9bdwqo6QF84R PGSSLMODE=verify-full PGSSLCERT=/home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.crt PGSSLKEY=/home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/replicator.key PGSSLROOTCERT=/home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01/root.crt /usr/lib/postgresql/17/bin/pg_basebackup -h db-identity-sau-main-dev-postgresql-worker-01.fastorder.com -p 5432 -U replicator -D /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01 -Fp -Xs -P -R --checkpoint=fast --wal-method=stream --verbose
pg_basebackup: initiating base backup, waiting for checkpoint to complete
pg_basebackup: checkpoint completed
pg_basebackup: write-ahead log start point: 0/2000028 on timeline 1
pg_basebackup: starting background WAL receiver
pg_basebackup: created temporary replication slot "pg_basebackup_1642367"
19466/30540 kB (63%), 0/1 tablespace (...er-01-standby-01/base/16384/3395)
30550/30550 kB (100%), 0/1 tablespace (...-01-standby-01/global/pg_control)
30550/30550 kB (100%), 1/1 tablespace
pg_basebackup: write-ahead log end point: 0/2000120
pg_basebackup: waiting for background process to finish streaming ...
pg_basebackup: syncing data to disk ...
pg_basebackup: renaming backup_manifest.tmp to backup_manifest
pg_basebackup: base backup completed
[0;32m[OK][0m pg_basebackup complete
[0;34m[INFO][0m Fixing postgresql.auto.conf to use IP-based primary_conninfo (matching golden backup)...
[2026-01-02 07:08:30 UTC] USER=www-data EUID=0 PID=1642372 ACTION=passthru ARGS=sudo -u postgres test -f /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01/standby.signal
[2026-01-02 07:08:30 UTC] USER=www-data EUID=0 PID=1642394 ACTION=fsop ARGS=chmod 600 /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01/standby.signal
[2026-01-02 07:08:30 UTC] USER=www-data EUID=0 PID=1642415 ACTION=fsop ARGS=chown postgres:postgres /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01/standby.signal
[2026-01-02 07:08:30 UTC] USER=www-data EUID=0 PID=1642424 ACTION=passthru ARGS=sudo -u postgres test -f /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01/standby.signal
[0;32m[OK][0m standby.signal verified and permissions set
[0;34m[INFO][0m Fixing postgresql.conf with standby-specific settings...
[1;33m[WARN][0m postgresql.conf not found at /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01/postgresql.conf
[0;34m[INFO][0m Verifying postgresql.auto.conf...
[1;33m[WARN][0m postgresql.auto.conf not found - pg_basebackup may have failed
[2026-01-02 07:08:30 UTC] USER=www-data EUID=0 PID=1642447 ACTION=fsop ARGS=rm -f /tmp/.pg_pwfile.4fJ0SB
[0;34m[INFO][0m Writing postgresql.conf (TLS≥1.2, SCRAM, audit logs)
[0;32m[OK][0m postgresql.conf updated successfully
[0;34m[INFO][0m Writing pg_hba.conf (mTLS with client certificates + SCRAM, least-privilege)
[2026-01-02 07:08:31 UTC] USER=www-data EUID=0 PID=1642496 ACTION=fsop ARGS=cp /tmp/tmp.ZYSYpVbGgO /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01/pg_hba.conf
[2026-01-02 07:08:31 UTC] USER=www-data EUID=0 PID=1642521 ACTION=fsop ARGS=chown postgres:postgres /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01/pg_hba.conf
[2026-01-02 07:08:31 UTC] USER=www-data EUID=0 PID=1642542 ACTION=fsop ARGS=chmod 600 /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01/pg_hba.conf
[0;32m[OK][0m pg_hba.conf updated
[0;34m[INFO][0m Creating systemd unit: /etc/systemd/system/postgresql@identity-sau-main-dev-worker-01-standby-01.service
[2026-01-02 07:08:31 UTC] USER=www-data EUID=0 PID=1642567 ACTION=fsop ARGS=mv -f /tmp/.pg_unit.bcJf9J /etc/systemd/system/postgresql@identity-sau-main-dev-worker-01-standby-01.service
[2026-01-02 07:08:31 UTC] USER=www-data EUID=0 PID=1642588 ACTION=fsop ARGS=chmod 0644 /etc/systemd/system/postgresql@identity-sau-main-dev-worker-01-standby-01.service
[0;32m[OK][0m systemd unit written
[2026-01-02 07:08:31 UTC] USER=www-data EUID=0 PID=1642610 ACTION=fsop ARGS=mkdir -p /var/lib/pgbackrest /var/spool/pgbackrest /var/log/pgbackrest
[2026-01-02 07:08:31 UTC] USER=www-data EUID=0 PID=1642632 ACTION=fsop ARGS=chown postgres:postgres /var/lib/pgbackrest /var/spool/pgbackrest /var/log/pgbackrest
[2026-01-02 07:08:31 UTC] USER=www-data EUID=0 PID=1642653 ACTION=passthru ARGS=systemctl daemon-reload
[0;34m[INFO][0m Starting PostgreSQL instance...
[2026-01-02 07:08:33 UTC] USER=www-data EUID=0 PID=1642788 ACTION=passthru ARGS=systemctl start postgresql@identity-sau-main-dev-worker-01-standby-01.service
[0;34m[INFO][0m Waiting for ACTIVE (systemd)…
[2026-01-02 07:08:34 UTC] USER=www-data EUID=0 PID=1642873 ACTION=passthru ARGS=systemctl is-active --quiet postgresql@identity-sau-main-dev-worker-01-standby-01.service
[0;32m[OK][0m Service ACTIVE
[0;34m[INFO][0m Waiting for port 5432 bind…
[0;32m[OK][0m Port bound
[0;34m[INFO][0m Waiting pg_isready (socket)…
[0;32m[OK][0m Readiness via socket OK
[0;34m[INFO][0m Waiting pg_isready (TCP db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com:5432)…
[0;32m[OK][0m Startup sequence complete
[0;34m[INFO][0m Configuring synchronous replication on primary worker-01...
[0;34m[INFO][0m Current synchronous_standby_names: ''
[0;34m[INFO][0m Initializing synchronous_standby_names with first standby
[0;34m[INFO][0m New synchronous_standby_names: 'ANY 1 (worker_01_standby_01)'
[2026-01-02 07:08:35 UTC] USER=www-data EUID=0 PID=1643019 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c ALTER SYSTEM SET synchronous_commit = on;
ALTER SYSTEM
[2026-01-02 07:08:35 UTC] USER=www-data EUID=0 PID=1643171 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -v ON_ERROR_STOP=1 -c ALTER SYSTEM SET synchronous_standby_names = 'ANY 1 (worker_01_standby_01)';
ALTER SYSTEM
[2026-01-02 07:08:36 UTC] USER=www-data EUID=0 PID=1643277 ACTION=passthru ARGS=sudo -u postgres psql -U postgres -h /var/run/postgresql-identity-sau-main-dev-worker-01 -p 5432 -d postgres --no-psqlrc -c SELECT pg_reload_conf();
pg_reload_conf
----------------
t
(1 row)
[0;32m[OK][0m ✅ Synchronous replication configured on primary
[0;32m[OK][0m Setting: ANY 1 (worker_01_standby_01)
[0;34m[INFO][0m Validating core security GUCs (via local socket)…
[0;32m[OK][0m Security GUCs verified (ssl, min TLS, SCRAM, audit logs)
[0;34m[INFO][0m Skipping database/role provisioning on standby node (read-only)
[0;34m[INFO][0m Database/roles will be replicated from primary: worker-01
[0;34m[INFO][0m Applying connection and memory optimizations...
[0;34m[INFO][0m Standby will use primary's max_connections: 100
[0;34m[INFO][0m Current settings: max_connections=100, work_mem=8MB
[0;34m[INFO][0m Target settings (standby): max_connections=100, work_mem=8MB
[0;32m[OK][0m Connection settings already optimized
[0;34m[INFO][0m Skipping password setting - this is a standby (read-only)
[0;34m[INFO][0m Use primary's postgres password to connect to this standby
[0;34m[INFO][0m Updating /etc/hosts with PostgreSQL hostname mappings...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m CONFIGURING POSTGRESQL NETWORK & DNS
[0;34m[INFO][0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: worker-01-standby-01
[0;34m[INFO][0m PostgreSQL IP: 10.100.1.211
[0;34m[INFO][0m Primary hostname: db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com
[0;34m[INFO][0m Adding /etc/hosts entry for worker-01-standby-01...
[0;34m[INFO][0m db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com → 10.100.1.211
[0;34m[INFO][0m ✅ db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com already exists with correct IP
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;32m✅[0m ✅ Network & DNS configuration complete
[0;32m✅[0m ═══════════════════════════════════════════════════════════════
[0;34m[INFO][0m Verifying /etc/hosts entries:
10.100.1.211 db-identity-sau-main-dev-postgresql-worker-01-standby-01.fastorder.com
[0;32m[OK][0m PostgreSQL 'identity-sau-main-dev' is up with TLS/SCRAM/logging.
Superuser TCP mode: cert
Connect (mTLS):
psql "sslmode=verify-full sslrootcert=/etc/fastorder/postgresql/certs/identity-sau-main-dev/worker-01-standby-01/ca.crt \
sslcert=/home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.crt \
sslkey=/home/postgres/ssl/.postgresql/identity-sau-main-dev/worker-01-standby-01/postgres.key \
host=db-identity-sau-main-dev-postgresql-worker-01-standby-01 port=5432 dbname=postgres user=postgres"
File been compeleted perfectly: 02-setup-pg-instance
[0;34m[INFO][0m Registering PostgreSQL node to observability API...
[INFO] Detected 4-part identifier format
[INFO] Registering node via API
[INFO] Application: PostgreSQL
[INFO] Identifier: identity-sau-main-dev-postgresql-worker-01-standby-01
[INFO] Identifier Parent: worker-01
[INFO] IP: 10.100.1.211
[INFO] Port: 5432
[INFO] FQDN: db-identity-sau-main-dev-postgresql-worker-01-standby-01
[INFO] Status: running
[INFO] Environment: identity-sau-main-dev (service=identity, zone=sau, branch=main, env=dev)
[INFO] Calling registration API: https://skeleton.dev.fastorder.com/api/obs/register
[SUCCESS] =========================================
[SUCCESS] Node registered successfully via API!
[SUCCESS] =========================================
[SUCCESS] Node UUID: 8eaa8059-bede-4f71-ae1d-d26590a898da
[SUCCESS] Environment UUID: 82a0dcd2-dcf2-422e-a830-b2dd51514393
[SUCCESS] Dashboard: https:\/\/skeleton.dev.fastorder.com\/dashboard\/monitoring\/environment\/82a0dcd2-dcf2-422e-a830-b2dd51514393
[0;32m[OK][0m PostgreSQL node registered to observability API
[DEBUG] Tracking substep start: steps/01-install/steps/03-role (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 03 role...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[2026-01-02 07:08:43 UTC] USER=www-data EUID=0 PID=1644164 ACTION=fsop ARGS=test -f /var/lib/postgresql/17/identity-sau-main-dev/worker-01-standby-01/standby.signal
⚠ This is a PostgreSQL STANDBY (read-only replica)
⚠ Skipping role creation - standby gets roles from primary via replication
⚠ Use the PRIMARY's credentials to connect to this standby
[DEBUG] Tracking substep start: steps/01-install/steps/05-setup-service (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 📦 05 setup service...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
ℹ️ Service-specific setup (identity) is handled by parent script
✅ Step 5 completed (service setup delegated to 01-install/run.sh)
🔍 DEBUG_CHECKPOINT_01: Starting service-specific steps discovery
🔍 DEBUG_CHECKPOINT_02: Searching for service folders in: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps
🔍 DEBUG_CHECKPOINT_03: Found directory: destroy
🔍 DEBUG_CHECKPOINT_03: Found directory: iam
🔍 DEBUG_CHECKPOINT_04: Found run.sh in: iam
🔍 DEBUG_CHECKPOINT_03: Found directory: identity
🔍 DEBUG_CHECKPOINT_04: Found run.sh in: identity
🔍 DEBUG_CHECKPOINT_03: Found directory: lib
🔍 DEBUG_CHECKPOINT_03: Found directory: passwords
🔍 DEBUG_CHECKPOINT_03: Found directory: role
🔍 DEBUG_CHECKPOINT_03: Found directory: ssl
🔍 DEBUG_CHECKPOINT_05: Service folders found: iam identity
[INFO] 📚 Detected service folders: iam identity
🔍 DEBUG_CHECKPOINT_06: Preparing to run service: iam at /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/iam/run.sh
[DEBUG] Tracking substep start: steps/01-install/steps/iam (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 🔸 Service: iam
🔍 DEBUG_CHECKPOINT_07: About to execute /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/iam/run.sh with IDENTIFIER=worker-01-standby-01 IDENTIFIER_PARENT=worker-01
🔍 DEBUG_CHECKPOINT_08: Running iam in AUTO mode
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[0;35m╔════════════════════════════════════════════════════════════════════════════╗[0m
[0;35m║ IAM Database Schema Initialization ║[0m
[0;35m╚════════════════════════════════════════════════════════════════════════════╝[0m
[0;34m[INFO][0m 🟢 Starting IAM schema provisioning...
[0;34m[INFO][0m Environment: identity-sau-main-dev
[0;34m[INFO][0m Identifier: coordinator
[0;34m[INFO][0m VM IP: 142.93.238.16
[0;34m[INFO][0m 📚 Discovered tables: core/01-tenant core/02-realm core/03-identity core/04-device core/05-identity_account core/06-identity_mfa core/07-external_idp_link policy/01-client policy/02-resource policy/03-scope policy/04-permission policy/05-role policy/06-role_permission policy/07-identity_role policy/08-policy_rule policy/09-api_key audit/01-auth_event audit/02-admin_action audit/03-risk_decision audit/04-consent_event
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m Schema: core[0m
[0;34m Core Identity Directory (tenants, realms, identities, devices, MFA)[0m
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m[INFO][0m 🔸 Table [1/20]: core/01-tenant
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.tenant Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Identifier: coordinator
Database: fastorder_identity_sau_main_dev_db
Host: db-identity-sau-main-dev-postgresql.fastorder.com:5432
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔐 Connecting to PostgreSQL over SSL (verify-full + mTLS)...
🗄️ Checking database: fastorder_identity_sau_main_dev_db
ℹ️ Database fastorder_identity_sau_main_dev_db already exists
✅ Connected to database: fastorder_identity_sau_main_dev_db
🔧 Installing extensions...
NOTICE: extension "uuid-ossp" already exists, skipping
CREATE EXTENSION
NOTICE: extension "pgcrypto" already exists, skipping
CREATE EXTENSION
NOTICE: extension "citext" already exists, skipping
CREATE EXTENSION
NOTICE: extension "dblink" already exists, skipping
CREATE EXTENSION
🔧 Installing Citus extension on coordinator...
NOTICE: extension "citus" already exists, skipping
CREATE EXTENSION
✅ Citus extension installed
✅ Extensions installed
🔧 Creating utils schema...
NOTICE: schema "utils" already exists, skipping
CREATE SCHEMA
✅ Utils schema created
🔧 Installing UUIDv7 function...
✅ UUIDv7 function installed
🔧 Creating core schema...
NOTICE: schema "core" already exists, skipping
CREATE SCHEMA
✅ Schema core created
🔧 Creating ENUM types...
DO
✅ ENUM types created
🔧 Creating core.tenant table...
NOTICE: relation "tenant" already exists, skipping
CREATE TABLE
COMMENT
COMMENT
COMMENT
✅ core.tenant created
🔧 Setting up Citus distribution for core.tenant...
✅ Citus distribution configured
🔧 Creating update trigger...
CREATE FUNCTION
ERROR: triggers are not supported on reference tables
ERROR: triggers are not supported on reference tables
✅ Update trigger created
✅ core.tenant initialization complete
[0;32m[OK][0m Table core/01-tenant initialized
[0;34m[INFO][0m 🔸 Table [2/20]: core/02-realm
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.realm Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.realm table...
NOTICE: relation "realm" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_realm_keycloak_id" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_realm_tenant" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ core.realm created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ core.realm initialization complete
[0;32m[OK][0m Table core/02-realm initialized
[0;34m[INFO][0m 🔸 Table [3/20]: core/03-identity
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.identity Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.identity table...
NOTICE: relation "identity" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_identity_unique_email" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_unique_keycloak" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_email" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_keycloak" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_status" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_type" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.identity created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ core.identity initialization complete
[0;32m[OK][0m Table core/03-identity initialized
[0;34m[INFO][0m 🔸 Table [4/20]: core/04-device
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.device Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.device table...
NOTICE: relation "device" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_device_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_device_fingerprint" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_device_trusted" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_device_last_seen" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.device created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ core.device initialization complete
[0;32m[OK][0m Table core/04-device initialized
[0;34m[INFO][0m 🔸 Table [5/20]: core/05-identity_account
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.identity_account Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.identity_account table...
NOTICE: relation "identity_account" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_identity_account_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_account_lockout" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_account_last_login" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.identity_account created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ core.identity_account initialization complete
[0;32m[OK][0m Table core/05-identity_account initialized
[0;34m[INFO][0m 🔸 Table [6/20]: core/06-identity_mfa
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.identity_mfa Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.identity_mfa table...
NOTICE: relation "identity_mfa" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_identity_mfa_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_mfa_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_mfa_active" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.identity_mfa created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ core.identity_mfa initialization complete
[0;32m[OK][0m Table core/06-identity_mfa initialized
[0;34m[INFO][0m 🔸 Table [7/20]: core/07-external_idp_link
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing core.external_idp_link Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating core.external_idp_link table...
NOTICE: relation "external_idp_link" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_external_idp_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_external_idp_provider" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_external_idp_email" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ core.external_idp_link created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ core.external_idp_link initialization complete
[0;32m[OK][0m Table core/07-external_idp_link initialized
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m Schema: policy[0m
[0;34m RBAC/ABAC Authorization (clients, roles, permissions, policies)[0m
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m[INFO][0m 🔸 Table [8/20]: policy/01-client
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.client Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy schema...
NOTICE: schema "policy" already exists, skipping
CREATE SCHEMA
✅ Schema policy created
🔧 Creating ENUM types...
DO
✅ ENUM types created
🔧 Creating policy.client table...
NOTICE: relation "client" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_client_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_client_keycloak" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_client_key" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_client_status" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ policy.client created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
CREATE FUNCTION
DROP TRIGGER
CREATE TRIGGER
✅ policy.client initialization complete
[0;32m[OK][0m Table policy/01-client initialized
[0;34m[INFO][0m 🔸 Table [9/20]: policy/02-resource
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.resource Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.resource table...
NOTICE: relation "resource" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_resource_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_resource_external" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_resource_owner" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.resource created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.resource initialization complete
[0;32m[OK][0m Table policy/02-resource initialized
[0;34m[INFO][0m 🔸 Table [10/20]: policy/03-scope
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.scope Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.scope table...
NOTICE: relation "scope" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_scope_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_scope_name" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.scope created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ policy.scope initialization complete
[0;32m[OK][0m Table policy/03-scope initialized
[0;34m[INFO][0m 🔸 Table [11/20]: policy/04-permission
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.permission Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.permission table...
NOTICE: relation "permission" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_permission_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_permission_name" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_permission_resource" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.permission created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ policy.permission initialization complete
[0;32m[OK][0m Table policy/04-permission initialized
[0;34m[INFO][0m 🔸 Table [12/20]: policy/05-role
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.role Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.role table...
NOTICE: relation "role" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_role_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_role_client" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_role_name" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_role_keycloak" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ policy.role created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ policy.role initialization complete
[0;32m[OK][0m Table policy/05-role initialized
[0;34m[INFO][0m 🔸 Table [13/20]: policy/06-role_permission
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.role_permission Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.role_permission table...
NOTICE: relation "role_permission" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_role_permission_role" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_role_permission_perm" already exists, skipping
CREATE INDEX
COMMENT
✅ policy.role_permission created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.role_permission initialization complete
[0;32m[OK][0m Table policy/06-role_permission initialized
[0;34m[INFO][0m 🔸 Table [14/20]: policy/07-identity_role
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.identity_role Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.identity_role table...
NOTICE: relation "identity_role" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_identity_role_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_role_role" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_role_active" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_identity_role_expires" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.identity_role created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.identity_role initialization complete
[0;32m[OK][0m Table policy/07-identity_role initialized
[0;34m[INFO][0m 🔸 Table [15/20]: policy/08-policy_rule
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.policy_rule Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.policy_rule table...
NOTICE: relation "policy_rule" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_policy_rule_realm" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_policy_rule_enabled" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_policy_rule_priority" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ policy.policy_rule created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
DROP TRIGGER
CREATE TRIGGER
✅ policy.policy_rule initialization complete
[0;32m[OK][0m Table policy/08-policy_rule initialized
[0;34m[INFO][0m 🔸 Table [16/20]: policy/09-api_key
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing policy.api_key Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating policy.api_key table...
NOTICE: relation "api_key" already exists, skipping
CREATE TABLE
NOTICE: relation "idx_api_key_prefix" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_api_key_client" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_api_key_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_api_key_status" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_api_key_expires" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
COMMENT
✅ policy.api_key created
🔧 Setting up Citus distribution...
✅ Citus distribution configured
✅ policy.api_key initialization complete
[0;32m[OK][0m Table policy/09-api_key initialized
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m Schema: audit[0m
[0;34m Audit & Risk Logging (auth events, admin actions, risk decisions)[0m
[0;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[0;34m[INFO][0m 🔸 Table [17/20]: audit/01-auth_event
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.auth_event Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Database: fastorder_identity_sau_main_dev_db
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit schema...
NOTICE: schema "audit" already exists, skipping
CREATE SCHEMA
✅ Schema audit created
🔧 Creating ENUM types...
DO
✅ ENUM types created
🔧 Creating audit.auth_event table...
NOTICE: relation "auth_event" already exists, skipping
CREATE TABLE
NOTICE: relation "audit.auth_event_2026_01" already exists, skipping
NOTICE: relation "audit.auth_event_2026_02" already exists, skipping
DO
NOTICE: relation "idx_auth_event_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_time" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_result" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_ip" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_session" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_trace" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_auth_event_risk" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ audit.auth_event created (partitioned)
✅ audit.auth_event initialization complete
[0;32m[OK][0m Table audit/01-auth_event initialized
[0;34m[INFO][0m 🔸 Table [18/20]: audit/02-admin_action
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.admin_action Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit.admin_action table...
NOTICE: relation "admin_action" already exists, skipping
CREATE TABLE
NOTICE: relation "audit.admin_action_2026_01" already exists, skipping
NOTICE: relation "audit.admin_action_2026_02" already exists, skipping
DO
NOTICE: relation "idx_admin_action_actor" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_admin_action_target" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_admin_action_time" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_admin_action_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_admin_action_trace" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ audit.admin_action created (partitioned)
✅ audit.admin_action initialization complete
[0;32m[OK][0m Table audit/02-admin_action initialized
[0;34m[INFO][0m 🔸 Table [19/20]: audit/03-risk_decision
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.risk_decision Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit.risk_decision table...
NOTICE: relation "risk_decision" already exists, skipping
CREATE TABLE
NOTICE: relation "audit.risk_decision_2026_01" already exists, skipping
NOTICE: relation "audit.risk_decision_2026_02" already exists, skipping
DO
NOTICE: relation "idx_risk_decision_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_risk_decision_level" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_risk_decision_decision" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_risk_decision_auth" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_risk_decision_time" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ audit.risk_decision created (partitioned)
✅ audit.risk_decision initialization complete
[0;32m[OK][0m Table audit/03-risk_decision initialized
[0;34m[INFO][0m 🔸 Table [20/20]: audit/04-consent_event
[0;34m[INFO][0m 📦 01 init schema...
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing audit.consent_event Table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔧 Creating audit.consent_event table...
NOTICE: relation "consent_event" already exists, skipping
CREATE TABLE
NOTICE: relation "audit.consent_event_2026_01" already exists, skipping
NOTICE: relation "audit.consent_event_2026_02" already exists, skipping
DO
NOTICE: relation "idx_consent_event_identity" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_consent_event_type" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_consent_event_version" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_consent_event_granted" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_consent_event_time" already exists, skipping
CREATE INDEX
COMMENT
COMMENT
✅ audit.consent_event created (partitioned)
🔧 Creating partition management functions...
CREATE FUNCTION
NOTICE: relation "audit.auth_event_2026_01" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_01
NOTICE: relation "audit.auth_event_2026_02" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_02
NOTICE: relation "audit.auth_event_2026_03" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_03
NOTICE: relation "audit.auth_event_2026_04" already exists, skipping
NOTICE: Created partition: audit.auth_event_2026_04
NOTICE: relation "audit.admin_action_2026_01" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_01
NOTICE: relation "audit.admin_action_2026_02" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_02
NOTICE: relation "audit.admin_action_2026_03" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_03
NOTICE: relation "audit.admin_action_2026_04" already exists, skipping
NOTICE: Created partition: audit.admin_action_2026_04
NOTICE: relation "audit.risk_decision_2026_01" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_01
NOTICE: relation "audit.risk_decision_2026_02" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_02
NOTICE: relation "audit.risk_decision_2026_03" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_03
NOTICE: relation "audit.risk_decision_2026_04" already exists, skipping
NOTICE: Created partition: audit.risk_decision_2026_04
NOTICE: relation "audit.consent_event_2026_01" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_01
NOTICE: relation "audit.consent_event_2026_02" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_02
NOTICE: relation "audit.consent_event_2026_03" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_03
NOTICE: relation "audit.consent_event_2026_04" already exists, skipping
NOTICE: Created partition: audit.consent_event_2026_04
create_monthly_partitions
---------------------------
(1 row)
CREATE VIEW
CREATE FUNCTION
COMMENT
COMMENT
✅ Partition management functions created
✅ audit.consent_event initialization complete
[0;32m[OK][0m Table audit/04-consent_event initialized
[0;35m════════════════════════════════════════════════════════════════════════════[0m
[0;32m[OK][0m ✅ IAM Schema Initialization Complete!
[0;32m[OK][0m All 20 tables initialized successfully
[0;34mSchemas created:[0m
• core - Identity directory (tenant, realm, identity, devices, MFA)
• policy - Authorization (clients, roles, permissions, policies, API keys)
• audit - Logging (auth events, admin actions, risk decisions, consent)
[0;34mDesign highlights:[0m
• Citus-ready with tenant_id distribution key
• NIST 800-63 identity compliance
• PCI DSS 4.0 audit logging
• GDPR consent tracking
• Keycloak integration via ID references
[0;35m════════════════════════════════════════════════════════════════════════════[0m
🔍 DEBUG_CHECKPOINT_06: Preparing to run service: identity at /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/run.sh
[DEBUG] Tracking substep start: steps/01-install/steps/identity (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
[INFO] 🔸 Service: identity
🔍 DEBUG_CHECKPOINT_07: About to execute /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/run.sh with IDENTIFIER=worker-01-standby-01 IDENTIFIER_PARENT=worker-01
🔍 DEBUG_CHECKPOINT_08: Running identity in AUTO mode
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[INFO] 🟢 Starting PostgreSQL provisioning for identity in sau-dev...
[INFO] Environment: identity-sau-main-dev
[INFO] Identifier: coordinator
[INFO] VM IP: 142.93.238.16
🔍 DEBUG_CHECKPOINT_A1: identity/run.sh started for SERVICE=identity
🔍 DEBUG_CHECKPOINT_A2: Checking SERVICE_ROOT: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity
🔍 DEBUG_CHECKPOINT_A3: SERVICE_ROOT exists, discovering table folders
🔍 DEBUG_CHECKPOINT_A4: Found subfolder: auth
🔍 DEBUG_CHECKPOINT_A4b: Checking for nested schema layout in: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth
🔍 DEBUG_CHECKPOINT_A4c: Found nested steps dir: /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps (display: auth/login)
🔍 DEBUG_CHECKPOINT_A5: Table step dirs discovered: auth/login|/opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps
🔍 DEBUG_CHECKPOINT_A6: Checking if we have table folders to process
[INFO] 📚 Detected grouped table folders under identity/: auth/login
🔍 DEBUG_CHECKPOINT_A7: Current IDENTIFIER=coordinator
🔍 DEBUG_CHECKPOINT_A8_PROCEED: Processing tables on coordinator/main node
🔍 DEBUG_CHECKPOINT_A9: Processing table: auth/login at /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps
[INFO] 🔸 Table group: auth/login
🔍 DEBUG_CHECKPOINT_A10: About to run numbered steps for table: auth/login
🔍 DEBUG_CHECKPOINT_B1: run_all_numbered_steps_in_dir called for dir=/opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps table=auth/login
🔍 DEBUG_CHECKPOINT_B2: Found 1 numbered steps: 01-init-schema.sh
🔍 DEBUG_CHECKPOINT_B3: About to run step: 01-init-schema.sh
Ab substep 0 compelete start
[DEBUG] Tracking substep start: steps/01-install/steps/identity/auth/login/01-init-schema (RUN_UUID=87f6d1b2-0166-4a82-a4fd-116a882b5cf3)
Ab substep 0 compelete start
[INFO] 📦 01 init schema...
Ab substep 1 compelete start
✓ Centralized Secrets Manager library loaded
Location: /opt/fastorder/bash/infra_core/secrets/secrets-vault.sh
Functions: PostgreSQL (build_pg_secret_name, get_pg_credentials, set_pg_credentials)
Elasticsearch (build_es_secret_name, get_es_credentials, set_es_credentials)
Provider: aws
🔑 Configuring AWS credentials...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Initializing auth.login_account table
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Identifier: coordinator
Database: fastorder_identity_sau_main_dev_db
Host: db-identity-sau-main-dev-postgresql.fastorder.com:5432
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
🔐 Connecting to PostgreSQL over SSL (verify-full + mTLS)...
🗄️ Checking database: fastorder_identity_sau_main_dev_db
ℹ️ Database fastorder_identity_sau_main_dev_db already exists
✅ Connected to database: fastorder_identity_sau_main_dev_db
ℹ️ Checking synchronous replication configuration...
synchronous_standby_names: ''
Connected standbys: 0
ℹ️ Synchronous replication not configured (standbys will be added later)
🔧 Installing extensions...
NOTICE: extension "uuid-ossp" already exists, skipping
CREATE EXTENSION
NOTICE: extension "dblink" already exists, skipping
CREATE EXTENSION
🔧 Installing Citus extension on coordinator...
NOTICE: extension "citus" already exists, skipping
CREATE EXTENSION
✅ Citus extension installed
✅ Extensions installed
🔧 Installing UUIDv7 function...
✅ UUIDv7 function installed
🔧 Creating auth schema...
NOTICE: schema "auth" already exists, skipping
CREATE SCHEMA
✅ Schema created
🔧 Creating account_status ENUM...
DO
✅ ENUM created
🔧 Creating auth.login_account table...
NOTICE: relation "login_account" already exists, skipping
CREATE TABLE
✅ Table created (Citus-compatible with region_hint in all constraints)
🔧 Creating indexes...
NOTICE: relation "idx_login_account_email" already exists, skipping
CREATE INDEX
NOTICE: relation "idx_login_account_username" already exists, skipping
CREATE INDEX
✅ Indexes created
ℹ️ Table already registered with Citus
🎉 Schema initialization complete for fastorder_identity_sau_main_dev_db
ℹ️ Skipping LISTEN/NOTIFY trigger on coordinator
CDC via Debezium is the primary change tracking mechanism
📊 Registering environment in monitoring database (obs schema)...
Topology: /opt/fastorder/bash/scripts/env_app_setup/state/identity-sau-main-dev/topology.json
Resource IP: 142.93.238.16
⚠️ Could not connect to monitoring database, skipping registration
You can manually register later using:
/opt/fastorder/bash/scripts/env_app_setup/setup/04-postgresql/steps/register-authN-af-aaaa1-dev.sh
==========================================
✅ Schema initialization complete!
==========================================
Ab substep 1 compelete end
Ab substep 2 compelete start
Ab substep 2 compelete end
🔍 DEBUG_CHECKPOINT_B4: Completed step: 01-init-schema.sh
🔍 DEBUG_CHECKPOINT_B5: All numbered steps completed for /opt/fastorder/bash/scripts/env_app_setup/setup/05-db/engine/postgresql/steps/01-install/steps/identity/../identity/auth/login/steps
🔍 DEBUG_CHECKPOINT_A11: Completed numbered steps for table: auth/login
compeleted here
🔍 DEBUG_CHECKPOINT_A12: All tables processed
End of 04-postgresql/steps/01-install/steps/identity/run.sh
[0;32m✓[0m ✅ Standby worker-01-standby-01 setup completed
[0;32m✓[0m ✅ PostgreSQL installation completed
[0;34m[INFO][0m Discovering additional setup steps...
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Executing step: 02-pg-bouncer.sh
[0;34m[INFO][0m ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[0;34m[INFO][0m Setting up PgBouncer connection pooling...
[2026-01-02 07:10:38 UTC] USER=www-data EUID=0 PID=1647876 ACTION=fsop ARGS=rm -f /tmp/pgbouncer-ip.service /tmp/pgbouncer.service
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
[0;32m✓ [SECRETS][0m Centralized Secrets Manager library loaded (Purpose-Engine Pattern)
[0;34m[SECRETS][0m Functions: PostgreSQL (build_pg_secret_name, get/set_pg_credentials_to_vault, rotate_pg_password)
[0;34m[SECRETS][0m Search (build_es_secret_name, get/set_es_credentials_to_vault)
[0;34m[SECRETS][0m Backups (build_backup_path)
[0;34m[SECRETS][0m Docs: /var/www/html/skeleton.dev.fastorder.com/docs/FASTCTL_USAGE_GUIDE.md
[0;34m[INFO][0m Checking for existing PgBouncer application environment in topology …
[0;32m[OK][0m Using existing PgBouncer environment:
[0;34m[INFO][0m IP: 10.100.1.204
[0;34m[INFO][0m FQDN: db-identity-sau-main-dev-postgresql-bouncer.fastorder.com
[0;34m[INFO][0m Domain: db-identity-sau-main-dev-postgresql-bouncer.fastorder.com
[0;34m[INFO][0m Ensuring /etc/hosts entry for db-identity-sau-main-dev-postgresql-bouncer.fastorder.com …
[0;34m[INFO][0m Adding db-identity-sau-main-dev-postgresql-bouncer.fastorder.com to /etc/hosts
[2026-01-02 07:10:39 UTC] USER=www-data EUID=0 PID=1647930 ACTION=fsop ARGS=sed -i /\sdb-identity-sau-main-dev-postgresql-bouncer.fastorder.com\(\s\|$\)/d /etc/hosts
[0;32m[OK][0m Added db-identity-sau-main-dev-postgresql-bouncer.fastorder.com -> 10.100.1.204 to /etc/hosts
[0;34m[INFO][0m Final verification of /etc/hosts entry for db-identity-sau-main-dev-postgresql-bouncer.fastorder.com …
[0;32m[OK][0m /etc/hosts correctly maps db-identity-sau-main-dev-postgresql-bouncer.fastorder.com to 10.100.1.204
[0;34m[INFO][0m Binding PgBouncer IP 10.100.1.204 to eth0:pgbouncer …
[2026-01-02 07:10:39 UTC] USER=www-data EUID=0 PID=1647964 ACTION=passthru ARGS=ip addr add 10.100.1.204/32 dev eth0 label eth0:pgbouncer
[0;32m[OK][0m Successfully bound 10.100.1.204 to eth0:pgbouncer
[2026-01-02 07:10:39 UTC] USER=www-data EUID=0 PID=1647982 ACTION=passthru ARGS=systemctl daemon-reload
[2026-01-02 07:10:41 UTC] USER=www-data EUID=0 PID=1648073 ACTION=passthru ARGS=systemctl restart pgbouncer-ip@identity-sau-main-dev.service
[2026-01-02 07:10:41 UTC] USER=www-data EUID=0 PID=1648083 ACTION=passthru ARGS=systemctl is-active --quiet pgbouncer-ip@identity-sau-main-dev.service
[0;32m[OK][0m pgbouncer-ip@identity-sau-main-dev.service is active
[2026-01-02 07:10:41 UTC] USER=www-data EUID=0 PID=1648107 ACTION=fsop ARGS=mkdir -p /etc/pgbouncer/identity-sau-main-dev
[2026-01-02 07:10:41 UTC] USER=www-data EUID=0 PID=1648116 ACTION=fsop ARGS=mkdir -p /run/pgbouncer/identity-sau-main-dev
[2026-01-02 07:10:41 UTC] USER=www-data EUID=0 PID=1648125 ACTION=fsop ARGS=mkdir -p /var/log/pgbouncer/identity-sau-main-dev
[2026-01-02 07:10:41 UTC] USER=www-data EUID=0 PID=1648134 ACTION=fsop ARGS=chmod 750 /etc/pgbouncer/identity-sau-main-dev
[2026-01-02 07:10:41 UTC] USER=www-data EUID=0 PID=1648143 ACTION=fsop ARGS=chmod 750 /run/pgbouncer/identity-sau-main-dev
[2026-01-02 07:10:42 UTC] USER=www-data EUID=0 PID=1648152 ACTION=fsop ARGS=chmod 750 /var/log/pgbouncer/identity-sau-main-dev
[2026-01-02 07:10:42 UTC] USER=www-data EUID=0 PID=1648161 ACTION=fsop ARGS=chown root:postgres /etc/pgbouncer/identity-sau-main-dev
[2026-01-02 07:10:42 UTC] USER=www-data EUID=0 PID=1648170 ACTION=fsop ARGS=chown postgres:postgres /run/pgbouncer/identity-sau-main-dev
[2026-01-02 07:10:42 UTC] USER=www-data EUID=0 PID=1648179 ACTION=fsop ARGS=chown postgres:postgres /var/log/pgbouncer/identity-sau-main-dev
[0;34m[INFO][0m Generating pgbouncer_admin client certificates...
[0;34m[INFO][0m ⏳ This may take 30-60 seconds...
[INFO] Loaded environment: identity-sau-main-dev (svc=identity zone=sau env=dev ip=142.93.238.16)
Environment: identity-sau-main-dev
Username: pgbouncer_admin
Identifier: pgbouncer
📦 Start executing client cert generation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Environment: identity-sau-main-dev
Service: identity
Zone: sau
Branch: main
Env: dev
Node: pgbouncer
User (CN): pgbouncer_admin
Hostname: db-identity-sau-main-dev-postgresql-bouncer.fastorder.com
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
[2026-01-02 07:10:42 UTC] USER=www-data EUID=0 PID=1648215 ACTION=fsop ARGS=chmod 755 /tmp/pg-client-pgbouncer-pgbouncer_admin
[2026-01-02 07:10:42 UTC] USER=www-data EUID=0 PID=1648224 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.crt /tmp/pg-client-pgbouncer-pgbouncer_admin/ra_root.crt
[2026-01-02 07:10:42 UTC] USER=www-data EUID=0 PID=1648233 ACTION=fsop ARGS=cp /opt/fastorder/ssl/ca/fastorder_ra_root/ra_root.key /tmp/pg-client-pgbouncer-pgbouncer_admin/ra_root.key
[2026-01-02 07:10:42 UTC] USER=www-data EUID=0 PID=1648242 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-pgbouncer-pgbouncer_admin/ra_root.crt
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648251 ACTION=fsop ARGS=chmod 644 /tmp/pg-client-pgbouncer-pgbouncer_admin/ra_root.key
🔑 Generating private key (PKCS#1 format)...
🔑 Converting to PKCS#8 PEM (for pgjdbc/debezium)...
🔑 (optional) Exporting DER as well...
📝 Generating CSR...
🔐 Signing with CA...
Certificate request self-signature ok
subject=CN = pgbouncer_admin
📂 Installing to canonical location → /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648267 ACTION=fsop ARGS=mkdir -p /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648276 ACTION=fsop ARGS=chmod 750 /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648285 ACTION=fsop ARGS=cp -f /tmp/pg-client-pgbouncer-pgbouncer_admin/pgbouncer_admin.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648294 ACTION=fsop ARGS=cp -f /tmp/pg-client-pgbouncer-pgbouncer_admin/pgbouncer_admin.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648303 ACTION=fsop ARGS=cp -f /tmp/pg-client-pgbouncer-pgbouncer_admin/ra_root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648312 ACTION=fsop ARGS=ln -sf root.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/ca.crt
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648321 ACTION=fsop ARGS=cp -f /tmp/pg-client-pgbouncer-pgbouncer_admin/pgbouncer_admin.key.pkcs1 /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648330 ACTION=fsop ARGS=cp -f /tmp/pg-client-pgbouncer-pgbouncer_admin/pgbouncer_admin_der.key /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648339 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648348 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt
/bin/chmod: cannot access '/etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt': Too many levels of symbolic links
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648357 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648366 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648375 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648384 ACTION=fsop ARGS=chown root:sslusers /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key
[2026-01-02 07:10:43 UTC] USER=www-data EUID=0 PID=1648393 ACTION=fsop ARGS=chown root:root /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt
/bin/chown: cannot dereference '/etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt': Too many levels of symbolic links
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648402 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt
/bin/chmod: cannot access '/etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt': Too many levels of symbolic links
✅ Canonical installation complete
📂 Creating symlinks for ab → /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648428 ACTION=fsop ARGS=mkdir -p /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648437 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648446 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648455 ACTION=fsop ARGS=chown ab:ab /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648464 ACTION=fsop ARGS=chmod 700 /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648473 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648482 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648491 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648500 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/ca.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/ca.crt
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648509 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1 /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648518 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648528 ACTION=fsop ARGS=chown -h ab:ssl-cert /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer/ca.crt
✅ Symlinks created for ab in /home/ab/ssl/.postgresql/identity-sau-main-dev/pgbouncer → /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer
📂 Creating symlinks for www-data → /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648538 ACTION=fsop ARGS=mkdir -p /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648548 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648558 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648567 ACTION=fsop ARGS=chown www-data:www-data /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648576 ACTION=fsop ARGS=chmod 700 /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648585 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648594 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt
[2026-01-02 07:10:44 UTC] USER=www-data EUID=0 PID=1648603 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648612 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/ca.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/ca.crt
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648621 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1 /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648630 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648640 ACTION=fsop ARGS=chown -h www-data:ssl-cert /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer/ca.crt
✅ Symlinks created for www-data in /home/www-data/ssl/.postgresql/identity-sau-main-dev/pgbouncer → /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer
📂 Creating symlinks for postgres → /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648650 ACTION=fsop ARGS=mkdir -p /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648659 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648668 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648677 ACTION=fsop ARGS=chown postgres:postgres /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648686 ACTION=fsop ARGS=chmod 700 /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648695 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648704 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648713 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648722 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/ca.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/ca.crt
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648731 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1 /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648740 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648750 ACTION=fsop ARGS=chown -h postgres:ssl-cert /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer/ca.crt
✅ Symlinks created for postgres in /home/postgres/ssl/.postgresql/identity-sau-main-dev/pgbouncer → /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer
📂 Creating symlinks for kafka → /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648760 ACTION=fsop ARGS=mkdir -p /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648769 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648778 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648787 ACTION=fsop ARGS=chown kafka:kafka /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648796 ACTION=fsop ARGS=chmod 700 /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648805 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648814 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648823 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648832 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/ca.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/ca.crt
[2026-01-02 07:10:45 UTC] USER=www-data EUID=0 PID=1648841 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1 /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key.pkcs1
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648850 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin_der.key
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648860 ACTION=fsop ARGS=chown -h kafka:ssl-cert /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/ca.crt
✅ Symlinks created for kafka in /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer → /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer
🎉 All requested users processed.
📋 Creating Kafka SSL certificate symlinks for www-data...
Source: /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem
Destination: /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648870 ACTION=fsop ARGS=mkdir -p /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648879 ACTION=fsop ARGS=chmod 750 /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648888 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
✅ Symlinked ca.pem
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648897 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
✅ Symlinked client-cert.pem
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648906 ACTION=fsop ARGS=ln -sf /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Symlinked client-key.pem
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648915 ACTION=fsop ARGS=chown -R www-data:www-data /var/www/ssl/kafka/identity-sau-main-dev
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648924 ACTION=fsop ARGS=chmod 640 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648933 ACTION=fsop ARGS=chown kafka:sslusers /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-key.pem
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648942 ACTION=fsop ARGS=chmod 644 /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/ca.pem /opt/kafka/secrets/identity-sau-main-dev/coordinator/pem/client-cert.pem
✅ Kafka certificate symlinks ready for www-data
PHP Kafka consumers can now use:
- ssl.ca.location: /var/www/ssl/kafka/identity-sau-main-dev/ca.pem
- ssl.certificate.location: /var/www/ssl/kafka/identity-sau-main-dev/client-cert.pem
- ssl.key.location: /var/www/ssl/kafka/identity-sau-main-dev/client-key.pem
✅ Client certificate generated successfully!
Environment: identity-sau-main-dev
User: pgbouncer_admin
Node: pgbouncer
FQDN: db-identity-sau-main-dev-postgresql-bouncer.fastorder.com
Next steps for Kafka Connect (Debezium → Postgres):
- Point connector to PEM key files:
database.sslcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt
database.sslkey: /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key # PKCS#8 PEM
database.sslrootcert: /home/kafka/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt
- If Connect runs in a container, bind-mount /home/kafka/ssl/.postgresql inside the container
and use the container path in connector config.
For local testing:
export PGSSLCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt"
export PGSSLKEY="/home/$USER/ssl/.postgresql/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key"
export PGSSLROOTCERT="/home/$USER/ssl/.postgresql/identity-sau-main-dev/pgbouncer/root.crt"
export PGSSLMODE="verify-full"
psql -h db-identity-sau-main-dev-postgresql-bouncer.fastorder.com -U pgbouncer_admin -d postgres
[0;32m[OK][0m mTLS client certificate present: /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt
[0;34m[INFO][0m Creating symlinks to canonical certificates in /etc/ssl/certs/identity-sau-main-dev/pg/pgbouncer-backend...
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648958 ACTION=fsop ARGS=mkdir -p /etc/ssl/certs/identity-sau-main-dev/pg/pgbouncer-backend
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648967 ACTION=fsop ARGS=mkdir -p /etc/ssl/private/identity-sau-main-dev/pg/pgbouncer-backend
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648976 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt /etc/ssl/certs/identity-sau-main-dev/pg/pgbouncer-backend/pgbouncer_admin.crt
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648985 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key /etc/ssl/private/identity-sau-main-dev/pg/pgbouncer-backend/pgbouncer_admin.key
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1648994 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt /etc/ssl/certs/identity-sau-main-dev/pg/pgbouncer-backend/root.crt
[0;34m[INFO][0m Creating coordinator CA symlink for PostgreSQL server verification...
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1649004 ACTION=fsop ARGS=ln -sf /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt /etc/ssl/certs/identity-sau-main-dev/pg/pgbouncer-backend/coordinator-ca.crt
[0;34m[INFO][0m Verifying canonical certificate permissions...
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1649013 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.crt
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1649022 ACTION=fsop ARGS=chmod 640 /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1649031 ACTION=fsop ARGS=chmod 644 /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt
/bin/chmod: cannot access '/etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/root.crt': Too many levels of symbolic links
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1649040 ACTION=fsop ARGS=chown root:www-data /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/pgbouncer_admin.key
[0;32m[OK][0m Backend certificate symlinks created in /etc/ssl
[0;32m[OK][0m Coordinator CA symlink created for server verification
[0;32m[OK][0m Certificates already in canonical location - no symlinks needed
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1649051 ACTION=fsop ARGS=test -r /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/server.crt
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1649060 ACTION=fsop ARGS=test -r /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/server.key
[2026-01-02 07:10:46 UTC] USER=www-data EUID=0 PID=1649069 ACTION=fsop ARGS=test -r /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/ca.crt
[0;31m[ERR][0m Missing/unreadable: /etc/fastorder/postgresql/certs/identity-sau-main-dev/pgbouncer/ca.crt
[0;31m[ERROR][0m ❌ Database infrastructure (postgresql) setup failed with exit code: 1
⏳ This step is pending and will execute after the previous steps complete successfully.
Loading logs...