diff --git a/Makefile b/Makefile deleted file mode 100644 index 3271003..0000000 --- a/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -# Makefile for PostgreSQL Service - -.PHONY: setup start stop restart status logs backup restore monitor clean help - -# Default environment -ENV ?= dev - -help: - @echo "PostgreSQL Service Management" - @echo "============================" - @echo "Available commands:" - @echo " make setup - Setup the PostgreSQL service (make scripts executable)" - @echo " make start - Start the PostgreSQL service" - @echo " make stop - Stop the PostgreSQL service" - @echo " make restart - Restart the PostgreSQL service" - @echo " make status - Check the status of the PostgreSQL service" - @echo " make logs - View the logs of the PostgreSQL service" - @echo " make backup - Create a backup of the PostgreSQL database" - @echo " make restore - Restore a backup of the PostgreSQL database" - @echo " make monitor - Monitor the PostgreSQL service" - @echo " make clean - Remove all containers, volumes, and networks" - @echo "" - @echo "Environment options:" - @echo " make start ENV=dev - Start with development configuration (default)" - @echo " make start ENV=staging - Start with staging configuration" - @echo " make start ENV=production - Start with production configuration" - -setup: - @echo "Setting up PostgreSQL service..." - @cd scripts && ./setup.sh - -start: - @echo "Starting PostgreSQL service with $(ENV) environment..." -ifeq ($(ENV), dev) - @docker-compose up -d -else - @docker-compose -f docker-compose.yaml -f environments/$(ENV).yaml up -d -endif - @echo "PostgreSQL service started." - -stop: - @echo "Stopping PostgreSQL service..." -ifeq ($(ENV), dev) - @docker-compose down -else - @docker-compose -f docker-compose.yaml -f environments/$(ENV).yaml down -endif - @echo "PostgreSQL service stopped." - -restart: stop start - -status: - @echo "PostgreSQL service status:" - @docker-compose ps - -logs: - @echo "PostgreSQL service logs:" - @docker-compose logs -f postgres - -backup: - @echo "Creating PostgreSQL backup..." - @cd scripts && ./backup.sh - -restore: - @echo "Restoring PostgreSQL backup..." - @cd scripts && ./restore.sh $(BACKUP) - -monitor: - @echo "Monitoring PostgreSQL service..." - @cd scripts && ./monitor.sh - -clean: - @echo "Cleaning up PostgreSQL service..." - @docker-compose down -v --remove-orphans - @echo "PostgreSQL service cleaned up." diff --git a/config/postgres-dev.conf b/config/postgres-dev.conf deleted file mode 100644 index 702a4ce..0000000 --- a/config/postgres-dev.conf +++ /dev/null @@ -1,62 +0,0 @@ -# PostgreSQL Development Configuration File - -# CONNECTIONS AND AUTHENTICATION -listen_addresses = '*' -max_connections = 50 -password_encryption = scram-sha-256 -ssl = off - -# RESOURCE USAGE -shared_buffers = 64MB -work_mem = 4MB -maintenance_work_mem = 32MB -effective_cache_size = 1GB -max_worker_processes = 4 -max_parallel_workers_per_gather = 1 -max_parallel_workers = 4 - -# WRITE-AHEAD LOG -wal_level = minimal -max_wal_size = 512MB -min_wal_size = 40MB -checkpoint_timeout = 5min -checkpoint_completion_target = 0.9 - -# QUERY TUNING -random_page_cost = 4.0 -effective_io_concurrency = 1 -default_statistics_target = 100 - -# LOGGING -log_destination = 'stderr' -logging_collector = on -log_directory = 'pg_log' -log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' -log_truncate_on_rotation = off -log_rotation_age = 1d -log_rotation_size = 10MB -log_min_duration_statement = 250 -log_checkpoints = on -log_connections = on -log_disconnections = on -log_duration = on -log_error_verbosity = verbose -log_line_prefix = '%m [%p] %q%u@%d ' -log_statement = 'all' - -# AUTOVACUUM -autovacuum = on -log_autovacuum_min_duration = 250 -autovacuum_max_workers = 2 -autovacuum_naptime = 1min -autovacuum_vacuum_threshold = 50 -autovacuum_analyze_threshold = 50 - -# CLIENT CONNECTION DEFAULTS -datestyle = 'iso, mdy' -timezone = 'UTC' -lc_messages = 'en_US.UTF-8' -lc_monetary = 'en_US.UTF-8' -lc_numeric = 'en_US.UTF-8' -lc_time = 'en_US.UTF-8' -default_text_search_config = 'pg_catalog.english' diff --git a/config/postgres-production.conf b/config/postgres-production.conf deleted file mode 100644 index 67e33c6..0000000 --- a/config/postgres-production.conf +++ /dev/null @@ -1,75 +0,0 @@ -# PostgreSQL Production Configuration File - -# CONNECTIONS AND AUTHENTICATION -listen_addresses = '*' -max_connections = 200 -password_encryption = scram-sha-256 -ssl = on -ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' -ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' - -# RESOURCE USAGE -shared_buffers = 2GB -work_mem = 8MB -maintenance_work_mem = 256MB -effective_cache_size = 8GB -max_worker_processes = 12 -max_parallel_workers_per_gather = 4 -max_parallel_workers = 12 - -# WRITE-AHEAD LOG -wal_level = replica -max_wal_size = 2GB -min_wal_size = 1GB -checkpoint_timeout = 15min -checkpoint_completion_target = 0.9 -archive_mode = on -archive_command = 'test ! -f /var/lib/postgresql/archive/%f && cp %p /var/lib/postgresql/archive/%f' - -# REPLICATION -max_wal_senders = 10 -wal_keep_size = 1GB -hot_standby = on -hot_standby_feedback = on - -# QUERY TUNING -random_page_cost = 1.1 -effective_io_concurrency = 200 -default_statistics_target = 500 -jit = on - -# LOGGING -log_destination = 'stderr' -logging_collector = on -log_directory = 'pg_log' -log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' -log_truncate_on_rotation = off -log_rotation_age = 1d -log_rotation_size = 100MB -log_min_duration_statement = 1000 -log_checkpoints = on -log_connections = on -log_disconnections = on -log_duration = off -log_error_verbosity = default -log_line_prefix = '%m [%p] %q%u@%d ' -log_statement = 'none' - -# AUTOVACUUM -autovacuum = on -log_autovacuum_min_duration = 1000 -autovacuum_max_workers = 6 -autovacuum_naptime = 1min -autovacuum_vacuum_threshold = 50 -autovacuum_analyze_threshold = 50 -autovacuum_vacuum_scale_factor = 0.05 -autovacuum_analyze_scale_factor = 0.025 - -# CLIENT CONNECTION DEFAULTS -datestyle = 'iso, mdy' -timezone = 'UTC' -lc_messages = 'en_US.UTF-8' -lc_monetary = 'en_US.UTF-8' -lc_numeric = 'en_US.UTF-8' -lc_time = 'en_US.UTF-8' -default_text_search_config = 'pg_catalog.english' diff --git a/config/postgres-staging.conf b/config/postgres-staging.conf deleted file mode 100644 index 6d5c4b1..0000000 --- a/config/postgres-staging.conf +++ /dev/null @@ -1,64 +0,0 @@ -# PostgreSQL Staging Configuration File - -# CONNECTIONS AND AUTHENTICATION -listen_addresses = '*' -max_connections = 100 -password_encryption = scram-sha-256 -ssl = on -ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' -ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' - -# RESOURCE USAGE -shared_buffers = 256MB -work_mem = 6MB -maintenance_work_mem = 64MB -effective_cache_size = 2GB -max_worker_processes = 6 -max_parallel_workers_per_gather = 2 -max_parallel_workers = 6 - -# WRITE-AHEAD LOG -wal_level = replica -max_wal_size = 1GB -min_wal_size = 80MB -checkpoint_timeout = 5min -checkpoint_completion_target = 0.9 - -# QUERY TUNING -random_page_cost = 4.0 -effective_io_concurrency = 2 -default_statistics_target = 100 - -# LOGGING -log_destination = 'stderr' -logging_collector = on -log_directory = 'pg_log' -log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' -log_truncate_on_rotation = off -log_rotation_age = 1d -log_rotation_size = 10MB -log_min_duration_statement = 500 -log_checkpoints = on -log_connections = on -log_disconnections = on -log_duration = off -log_error_verbosity = default -log_line_prefix = '%m [%p] %q%u@%d ' -log_statement = 'mod' - -# AUTOVACUUM -autovacuum = on -log_autovacuum_min_duration = 500 -autovacuum_max_workers = 3 -autovacuum_naptime = 1min -autovacuum_vacuum_threshold = 50 -autovacuum_analyze_threshold = 50 - -# CLIENT CONNECTION DEFAULTS -datestyle = 'iso, mdy' -timezone = 'UTC' -lc_messages = 'en_US.UTF-8' -lc_monetary = 'en_US.UTF-8' -lc_numeric = 'en_US.UTF-8' -lc_time = 'en_US.UTF-8' -default_text_search_config = 'pg_catalog.english' diff --git a/config/postgres.conf b/config/postgres.conf index 3fd1515..054c38c 100644 --- a/config/postgres.conf +++ b/config/postgres.conf @@ -1,34 +1,47 @@ -# PostgreSQL Configuration File -# This is the main configuration file that will be used by default +# PostgreSQL Production Configuration File for Proxmox LXC Container +# Optimized for production workloads # CONNECTIONS AND AUTHENTICATION listen_addresses = '*' -max_connections = 100 +max_connections = 200 # Increased for production workloads password_encryption = scram-sha-256 ssl = on ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' +ssl_prefer_server_ciphers = on # Prefer server's cipher order +ssl_min_protocol_version = 'TLSv1.2' # Minimum TLS version -# RESOURCE USAGE -shared_buffers = 128MB -work_mem = 4MB -maintenance_work_mem = 64MB -effective_cache_size = 4GB -max_worker_processes = 8 -max_parallel_workers_per_gather = 2 -max_parallel_workers = 8 +# RESOURCE USAGE (Optimized for 16GB RAM, 4 CPUs) +shared_buffers = 4GB # 25% of system memory for 16GB RAM +work_mem = 64MB # Increased for better query performance +maintenance_work_mem = 1GB # Increased for faster maintenance operations +effective_cache_size = 12GB # 75% of available memory +max_worker_processes = 8 # 2 × CPU cores +max_parallel_workers_per_gather = 4 # 1 per CPU core +max_parallel_workers = 8 # Equal to max_worker_processes +huge_pages = try # Try to use huge pages if available # WRITE-AHEAD LOG -wal_level = replica -max_wal_size = 1GB -min_wal_size = 80MB -checkpoint_timeout = 5min -checkpoint_completion_target = 0.9 +wal_level = replica # Enables WAL archiving and replication +max_wal_size = 2GB # Increased for production +min_wal_size = 1GB # Increased for production +checkpoint_timeout = 15min # Increased to reduce checkpoint frequency +checkpoint_completion_target = 0.9 # Spread checkpoint I/O over more time +wal_buffers = 16MB # Increased for better performance +synchronous_commit = on # Ensures data durability # QUERY TUNING -random_page_cost = 4.0 -effective_io_concurrency = 2 -default_statistics_target = 100 +random_page_cost = 1.1 # Optimized for SSD storage +effective_io_concurrency = 200 # Increased for SSD storage +default_statistics_target = 500 # Increased for better query plans +jit = on # Enable JIT compilation for better performance +track_io_timing = on # Track I/O timing for better insights + +# REPLICATION +max_wal_senders = 10 # Allow up to 10 WAL sender processes +wal_keep_size = 1GB # Keep at least 1GB of WAL segments +hot_standby = on # Allow queries during recovery +hot_standby_feedback = on # Prevent query conflicts with standby servers # LOGGING log_destination = 'stderr' @@ -37,23 +50,32 @@ log_directory = 'pg_log' log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' log_truncate_on_rotation = off log_rotation_age = 1d -log_rotation_size = 10MB -log_min_duration_statement = 1000 +log_rotation_size = 100MB # Increased for production +log_min_duration_statement = 1000 # Log slow queries (1 second) log_checkpoints = on log_connections = on log_disconnections = on log_duration = off log_error_verbosity = default log_line_prefix = '%m [%p] %q%u@%d ' -log_statement = 'none' +log_statement = 'none' # Don't log statements in production +log_temp_files = 0 # Log all temp file usage # AUTOVACUUM autovacuum = on log_autovacuum_min_duration = 1000 -autovacuum_max_workers = 3 +autovacuum_max_workers = 6 # Increased for production autovacuum_naptime = 1min autovacuum_vacuum_threshold = 50 autovacuum_analyze_threshold = 50 +autovacuum_vacuum_scale_factor = 0.05 # More aggressive vacuuming +autovacuum_analyze_scale_factor = 0.025 # More aggressive analyzing + +# STATISTICS +track_activities = on +track_counts = on +track_functions = all # Track function statistics +track_activity_query_size = 4096 # Increased for better monitoring # CLIENT CONNECTION DEFAULTS datestyle = 'iso, mdy' @@ -63,3 +85,12 @@ lc_monetary = 'en_US.UTF-8' lc_numeric = 'en_US.UTF-8' lc_time = 'en_US.UTF-8' default_text_search_config = 'pg_catalog.english' + +# LOCKS AND DEADLOCKS +deadlock_timeout = 1s # Check for deadlocks after 1s + +# DEVELOPER OPTIONS +debug_print_parse = off +debug_print_rewritten = off +debug_print_plan = off +debug_pretty_print = on diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml deleted file mode 100644 index 6bd1f0d..0000000 --- a/docker-compose.override.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# This override file is for local development and is automatically loaded when running docker-compose up -# without specifying a different file - -services: - postgres: - ports: - - "5432:5432" - volumes: - - ./config/postgres-dev.conf:/etc/postgresql/postgresql.conf - environment: - POSTGRES_PASSWORD: postgres_dev_password - command: postgres -c config_file=/etc/postgresql/postgresql.conf - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] - interval: 5s - timeout: 3s - retries: 3 - start_period: 10s - - pgadmin: - ports: - - "5050:80" - environment: - PGADMIN_DEFAULT_EMAIL: dev@example.com - PGADMIN_DEFAULT_PASSWORD: pgadmin_dev_password diff --git a/environments/.env.production b/environments/.env.production new file mode 100644 index 0000000..e41f6bc --- /dev/null +++ b/environments/.env.production @@ -0,0 +1,25 @@ +# PostgreSQL Production Environment Configuration + +# PostgreSQL Settings +POSTGRES_USER=postgres +POSTGRES_PASSWORD=secure_production_password +POSTGRES_DB=postgres + +# Resource Limits +POSTGRES_CPU_LIMIT=2 +POSTGRES_MEMORY_LIMIT=4G + +# Storage Configuration +POSTGRES_DATA_PATH=/data/postgres + +# Network Configuration +POSTGRES_SUBNET=172.28.0.0/16 +POSTGRES_NETWORK=postgres_network_prod + +# Logging Configuration +POSTGRES_LOG_MAX_SIZE=200m +POSTGRES_LOG_MAX_FILE=10 + +# Backup Configuration +BACKUP_RETENTION_DAYS=30 +BACKUP_DIR=/backups/postgres diff --git a/environments/dev.yaml b/environments/dev.yaml deleted file mode 100644 index 9d58d1e..0000000 --- a/environments/dev.yaml +++ /dev/null @@ -1,33 +0,0 @@ -version: '3.8' - -services: - postgres: - environment: - POSTGRES_USER: postgres_dev - POSTGRES_PASSWORD: postgres_dev_password - POSTGRES_DB: postgres_dev - ports: - - "5432:5432" - volumes: - - postgres_data_dev:/var/lib/postgresql/data - - ./config/postgres-dev.conf:/etc/postgresql/postgresql.conf - command: postgres -c config_file=/etc/postgresql/postgresql.conf - - pgadmin: - environment: - PGADMIN_DEFAULT_EMAIL: dev@example.com - PGADMIN_DEFAULT_PASSWORD: pgadmin_dev_password - ports: - - "5050:80" - volumes: - - pgadmin_data_dev:/var/lib/pgadmin - -volumes: - postgres_data_dev: - name: postgres_data_dev - pgadmin_data_dev: - name: pgadmin_data_dev - -networks: - postgres_network: - name: postgres_network_dev diff --git a/environments/production.yaml b/environments/production.yaml deleted file mode 100644 index 53432ad..0000000 --- a/environments/production.yaml +++ /dev/null @@ -1,82 +0,0 @@ -version: '3.8' - -services: - postgres: - image: postgres:15-alpine - environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB} - ports: - - "5432:5432" - volumes: - - postgres_data_prod:/var/lib/postgresql/data - - ./config/postgres-production.conf:/etc/postgresql/postgresql.conf - command: postgres -c config_file=/etc/postgresql/postgresql.conf - deploy: - resources: - limits: - cpus: '2' - memory: 4G - restart_policy: - condition: any - delay: 5s - max_attempts: 3 - window: 120s - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "10" - - pgadmin: - image: dpage/pgadmin4:latest - environment: - PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL} - PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD} - PGADMIN_CONFIG_SERVER_MODE: 'True' - PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED: 'True' - ports: - - "5050:80" - volumes: - - pgadmin_data_prod:/var/lib/pgadmin - deploy: - resources: - limits: - cpus: '1' - memory: 1G - restart_policy: - condition: any - delay: 5s - max_attempts: 3 - window: 120s - logging: - driver: "json-file" - options: - max-size: "100m" - max-file: "5" - -volumes: - postgres_data_prod: - name: postgres_data_prod - driver: local - driver_opts: - type: 'none' - o: 'bind' - device: '/data/postgres' - pgadmin_data_prod: - name: pgadmin_data_prod - driver: local - driver_opts: - type: 'none' - o: 'bind' - device: '/data/pgadmin' - -networks: - postgres_network: - name: postgres_network_prod - driver: bridge - ipam: - driver: default - config: - - subnet: 172.28.0.0/16 diff --git a/environments/staging.yaml b/environments/staging.yaml deleted file mode 100644 index 46481e2..0000000 --- a/environments/staging.yaml +++ /dev/null @@ -1,43 +0,0 @@ -version: '3.8' - -services: - postgres: - environment: - POSTGRES_USER: postgres_staging - POSTGRES_PASSWORD: postgres_staging_password - POSTGRES_DB: postgres_staging - ports: - - "5433:5432" - volumes: - - postgres_data_staging:/var/lib/postgresql/data - - ./config/postgres-staging.conf:/etc/postgresql/postgresql.conf - command: postgres -c config_file=/etc/postgresql/postgresql.conf - deploy: - resources: - limits: - cpus: '1' - memory: 1G - - pgadmin: - environment: - PGADMIN_DEFAULT_EMAIL: staging@example.com - PGADMIN_DEFAULT_PASSWORD: pgadmin_staging_password - ports: - - "5051:80" - volumes: - - pgadmin_data_staging:/var/lib/pgadmin - deploy: - resources: - limits: - cpus: '0.5' - memory: 512M - -volumes: - postgres_data_staging: - name: postgres_data_staging - pgadmin_data_staging: - name: pgadmin_data_staging - -networks: - postgres_network: - name: postgres_network_staging diff --git a/proxmox-lxc-setup.md b/proxmox-lxc-setup.md index 3036aab..5e4beb2 100644 --- a/proxmox-lxc-setup.md +++ b/proxmox-lxc-setup.md @@ -255,7 +255,7 @@ git clone ssh://git@gitea.mehmetkaratay.com.tr:222/evyos-center-server/postgres- 3. Create a proper .env file with secure credentials: ```bash -cp .env.example .env +cp environments/.env.production .env nano .env ``` @@ -265,13 +265,19 @@ nano .env chmod +x scripts/*.sh ``` -5. Start the PostgreSQL service: +5. Create the Docker network for PostgreSQL: + +```bash +docker network create postgres_network +``` + +6. Start the PostgreSQL service: ```bash docker-compose up --build -d ``` -6. Verify that the containers are running: +7. Verify that the container is running: ```bash docker-compose ps @@ -334,24 +340,67 @@ If you encounter issues: ## Maintenance -1. Backup your data regularly: +### Backup and Restore + +1. Create a backup of your PostgreSQL database: ```bash + cd /opt/postgres-service ./scripts/backup.sh ``` -2. Update your containers: +2. Restore from a backup when needed: ```bash + cd /opt/postgres-service + ./scripts/restore.sh backups/postgres_backup_YYYYMMDD_HHMMSS.sql.gz + ``` + +### Monitoring + +1. Monitor PostgreSQL performance: + ```bash + cd /opt/postgres-service + ./scripts/monitor.sh + ``` + +2. Check PostgreSQL logs: + ```bash + docker-compose logs postgres + ``` + +3. Monitor system resources: + ```bash + htop + ``` + +4. Check disk usage: + ```bash + df -h + ``` + +### Updates and Maintenance + +1. Update PostgreSQL container: + ```bash + cd /opt/postgres-service docker-compose pull docker-compose down docker-compose up -d ``` -3. Monitor your system resources: +2. Restart PostgreSQL service: ```bash - htop + cd /opt/postgres-service + docker-compose restart ``` -4. Monitor PostgreSQL performance: +3. Stop PostgreSQL service: ```bash - ./scripts/monitor.sh + cd /opt/postgres-service + docker-compose down + ``` + +4. Start PostgreSQL service: + ```bash + cd /opt/postgres-service + docker-compose up -d ```