338 lines
13 KiB
YAML
338 lines
13 KiB
YAML
---
|
|
# Data Service Provisioning Playbook
|
|
#
|
|
# Provisions PostgreSQL database, Valkey ACL user, Garage S3 bucket/key,
|
|
# and Vault credentials for a service defined in services.yml.
|
|
#
|
|
# Usage:
|
|
# ansible-playbook -i inventory.ini playbooks/data-service.yml -e "service=myapp"
|
|
#
|
|
# With database restore:
|
|
# ansible-playbook -i inventory.ini playbooks/data-service.yml -e "service=myapp" -e "restore=true"
|
|
#
|
|
# Prerequisites:
|
|
# - postgres-primary running (run playbooks/postgres.yml first)
|
|
# - valkey-primary running with ACLs (run playbooks/valkey.yml first)
|
|
# - Vault cluster initialized and unsealed (run playbooks/vault.yml first)
|
|
# - Database secrets engine enabled: vault secrets enable database
|
|
# - VAULT_ADDR and VAULT_TOKEN environment variables set
|
|
|
|
- name: Load Service Configuration
|
|
hosts: localhost
|
|
gather_facts: false
|
|
vars_files:
|
|
- ../services.yml
|
|
tasks:
|
|
- name: Validate service parameter
|
|
fail:
|
|
msg: "Service '{{ service }}' not found in services.yml"
|
|
when: service not in services
|
|
|
|
- name: Set service facts
|
|
set_fact:
|
|
svc: "{{ services[service] }}"
|
|
postgres_enabled: "{{ services[service].postgres.enabled | default(false) }}"
|
|
valkey_enabled: "{{ services[service].valkey.enabled | default(false) }}"
|
|
s3_enabled: "{{ services[service].s3.enabled | default(false) }}"
|
|
vault_roles: "{{ services[service].vault_roles | default(['app', 'migrate']) }}"
|
|
|
|
- name: Display service info
|
|
debug:
|
|
msg: |
|
|
Service: {{ service }}
|
|
Description: {{ svc.description }}
|
|
PostgreSQL: {{ postgres_enabled }}
|
|
Valkey: {{ valkey_enabled }} (prefix: {{ svc.valkey.key_prefix | default(service) }}:*)
|
|
S3: {{ s3_enabled }} (bucket: {{ svc.s3.bucket | default(service + '-media') }})
|
|
Vault roles: {{ vault_roles | join(', ') }}
|
|
|
|
- name: Setup PostgreSQL Database and Roles
|
|
hosts: postgres-01
|
|
become: true
|
|
vars_files:
|
|
- ../vault/secrets.yml
|
|
- ../services.yml
|
|
vars:
|
|
svc: "{{ services[service] }}"
|
|
tasks:
|
|
- name: Skip if PostgreSQL not enabled
|
|
meta: end_host
|
|
when: not (svc.postgres.enabled | default(false))
|
|
|
|
- name: Check if database exists
|
|
become_user: postgres
|
|
shell: psql -tAc "SELECT 1 FROM pg_database WHERE datname='{{ service }}'"
|
|
register: db_exists
|
|
changed_when: false
|
|
|
|
- name: Template static roles SQL
|
|
template:
|
|
src: ../templates/pg-static-roles.sql.j2
|
|
dest: "/tmp/{{ service }}-roles.sql"
|
|
mode: '0644'
|
|
when: db_exists.stdout != "1"
|
|
|
|
- name: Create database and static roles
|
|
become_user: postgres
|
|
shell: psql -f /tmp/{{ service }}-roles.sql
|
|
when: db_exists.stdout != "1"
|
|
|
|
- name: Create common extensions (requires superuser)
|
|
become_user: postgres
|
|
shell: |
|
|
psql -d {{ service }} -c "CREATE EXTENSION IF NOT EXISTS btree_gist;"
|
|
psql -d {{ service }} -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";'
|
|
when: db_exists.stdout != "1"
|
|
|
|
- name: Clean up SQL file
|
|
file:
|
|
path: "/tmp/{{ service }}-roles.sql"
|
|
state: absent
|
|
|
|
- name: Check for dump file
|
|
delegate_to: localhost
|
|
become: false
|
|
stat:
|
|
path: "{{ playbook_dir }}/../{{ svc.postgres.restore_from }}"
|
|
register: dump_file
|
|
when: restore | default(false) | bool
|
|
|
|
- name: Copy dump to server
|
|
copy:
|
|
src: "{{ playbook_dir }}/../{{ svc.postgres.restore_from }}"
|
|
dest: "/tmp/{{ service }}.dump"
|
|
mode: '0644'
|
|
when:
|
|
- restore | default(false) | bool
|
|
- dump_file.stat.exists | default(false)
|
|
|
|
- name: Restore database from dump
|
|
become_user: postgres
|
|
shell: pg_restore --no-owner --no-privileges -d {{ service }} /tmp/{{ service }}.dump
|
|
when:
|
|
- restore | default(false) | bool
|
|
- dump_file.stat.exists | default(false)
|
|
ignore_errors: true # May fail if data already exists
|
|
|
|
- name: Clean up dump file
|
|
file:
|
|
path: "/tmp/{{ service }}.dump"
|
|
state: absent
|
|
when: restore | default(false) | bool
|
|
|
|
- name: Setup Valkey ACL User
|
|
hosts: valkey-01
|
|
become: true
|
|
vars_files:
|
|
- ../vault/secrets.yml
|
|
- ../services.yml
|
|
vars:
|
|
svc: "{{ services[service] }}"
|
|
valkey_nebula_ip: "{{ hostvars['valkey-01']['nebula_ip'] }}"
|
|
tasks:
|
|
- name: Skip if Valkey not enabled
|
|
meta: end_host
|
|
when: not (svc.valkey.enabled | default(false))
|
|
|
|
- name: Generate service password
|
|
set_fact:
|
|
valkey_service_password: "{{ lookup('password', '/dev/null length=32 chars=hexdigits') }}"
|
|
|
|
- name: Check if ACL user exists
|
|
command: valkey-cli -h {{ valkey_nebula_ip }} --user admin --pass {{ valkey_admin_password }} ACL GETUSER {{ service }}
|
|
register: acl_user_check
|
|
changed_when: false
|
|
failed_when: false
|
|
no_log: true
|
|
|
|
- name: Create ACL user for service
|
|
shell: |
|
|
valkey-cli -h {{ valkey_nebula_ip }} --user admin --pass {{ valkey_admin_password }} \
|
|
ACL SETUSER {{ service }} on '>{{ valkey_service_password }}' '~{{ svc.valkey.key_prefix | default(service) }}:*' '&*' '+@all'
|
|
when: acl_user_check.rc != 0
|
|
no_log: true
|
|
|
|
- name: Update ACL user password if exists
|
|
shell: |
|
|
valkey-cli -h {{ valkey_nebula_ip }} --user admin --pass {{ valkey_admin_password }} \
|
|
ACL SETUSER {{ service }} on '>{{ valkey_service_password }}' '~{{ svc.valkey.key_prefix | default(service) }}:*' '&*' '+@all'
|
|
when: acl_user_check.rc == 0
|
|
no_log: true
|
|
|
|
- name: Persist ACL to disk
|
|
command: valkey-cli -h {{ valkey_nebula_ip }} --user admin --pass {{ valkey_admin_password }} ACL SAVE
|
|
no_log: true
|
|
|
|
- name: Store credentials in Vault
|
|
delegate_to: localhost
|
|
become: false
|
|
shell: |
|
|
vault kv put secret/{{ service }}/valkey \
|
|
host={{ valkey_nebula_ip }} \
|
|
port=6379 \
|
|
username={{ service }} \
|
|
password={{ valkey_service_password }} \
|
|
key_prefix={{ svc.valkey.key_prefix | default(service) }}
|
|
environment:
|
|
VAULT_ADDR: "{{ lookup('env', 'VAULT_ADDR') | default('http://' + hostvars['vault-01']['nebula_ip'] + ':8200', true) }}"
|
|
VAULT_TOKEN: "{{ lookup('env', 'VAULT_TOKEN') }}"
|
|
no_log: true
|
|
|
|
- name: Setup Garage S3 Bucket and Key
|
|
hosts: garage-01
|
|
become: true
|
|
vars_files:
|
|
- ../services.yml
|
|
vars:
|
|
svc: "{{ services[service] }}"
|
|
garage_nebula_ip: "{{ hostvars['garage-01']['nebula_ip'] }}"
|
|
tasks:
|
|
- name: Skip if S3 not enabled
|
|
meta: end_host
|
|
when: not (svc.s3.enabled | default(false))
|
|
|
|
- name: Set bucket name
|
|
set_fact:
|
|
bucket_name: "{{ svc.s3.bucket | default(service + '-media') }}"
|
|
|
|
- name: Check if bucket exists
|
|
command: garage -c /etc/garage/garage.toml bucket list
|
|
register: bucket_list
|
|
changed_when: false
|
|
|
|
- name: Create bucket if needed
|
|
command: garage -c /etc/garage/garage.toml bucket create {{ bucket_name }}
|
|
when: bucket_name not in bucket_list.stdout
|
|
|
|
- name: Check if key exists
|
|
command: garage -c /etc/garage/garage.toml key list
|
|
register: key_list
|
|
changed_when: false
|
|
|
|
- name: Create API key for service
|
|
command: garage -c /etc/garage/garage.toml key create {{ service }}-key
|
|
register: key_create
|
|
when: (service + '-key') not in key_list.stdout
|
|
|
|
- name: Get key info
|
|
command: garage -c /etc/garage/garage.toml key info {{ service }}-key --show-secret
|
|
register: key_info
|
|
changed_when: false
|
|
no_log: true
|
|
|
|
- name: Parse key credentials
|
|
set_fact:
|
|
s3_access_key: "{{ key_info.stdout | regex_search('Key ID: ([A-Za-z0-9]+)', '\\1') | first }}"
|
|
s3_secret_key: "{{ key_info.stdout | regex_search('Secret key: ([a-f0-9]+)', '\\1') | first }}"
|
|
no_log: true
|
|
|
|
- name: Grant bucket permissions to key
|
|
command: >
|
|
garage -c /etc/garage/garage.toml bucket allow {{ bucket_name }}
|
|
--read --write --key {{ service }}-key
|
|
register: bucket_allow
|
|
changed_when: "'already' not in bucket_allow.stderr"
|
|
|
|
- name: Store S3 credentials in Vault
|
|
delegate_to: localhost
|
|
become: false
|
|
shell: |
|
|
vault kv put secret/{{ service }}/s3 \
|
|
access_key={{ s3_access_key }} \
|
|
secret_key={{ s3_secret_key }} \
|
|
bucket={{ bucket_name }} \
|
|
endpoint=http://{{ garage_nebula_ip }}:3900
|
|
environment:
|
|
VAULT_ADDR: "{{ lookup('env', 'VAULT_ADDR') | default('http://' + hostvars['vault-01']['nebula_ip'] + ':8200', true) }}"
|
|
VAULT_TOKEN: "{{ lookup('env', 'VAULT_TOKEN') }}"
|
|
no_log: true
|
|
|
|
- name: Configure Vault Database Credentials
|
|
hosts: localhost
|
|
gather_facts: false
|
|
vars_files:
|
|
- ../vault/secrets.yml
|
|
- ../services.yml
|
|
vars:
|
|
svc: "{{ services[service] }}"
|
|
postgres_nebula_ip: "{{ hostvars['postgres-01']['nebula_ip'] }}"
|
|
vault_nebula_ip: "{{ hostvars['vault-01']['nebula_ip'] }}"
|
|
environment:
|
|
VAULT_ADDR: "{{ vault_addr | default('http://' + vault_nebula_ip + ':8200') }}"
|
|
tasks:
|
|
- name: Skip if PostgreSQL not enabled
|
|
meta: end_play
|
|
when: not (svc.postgres.enabled | default(false))
|
|
|
|
- name: Check if VAULT_TOKEN is set
|
|
fail:
|
|
msg: "VAULT_TOKEN environment variable must be set"
|
|
when: lookup('env', 'VAULT_TOKEN') == ''
|
|
|
|
- name: Configure Vault database connection
|
|
shell: |
|
|
vault write database/config/{{ service }} \
|
|
plugin_name="postgresql-database-plugin" \
|
|
allowed_roles="{{ service }}-app,{{ service }}-migrate" \
|
|
connection_url="postgresql://{% raw %}{{username}}:{{password}}{% endraw %}@{{ postgres_nebula_ip }}:5432/{{ service }}" \
|
|
username="vault_admin" \
|
|
password="{{ vault_admin_password }}"
|
|
register: vault_config
|
|
changed_when: vault_config.rc == 0
|
|
|
|
- name: Create Vault app role
|
|
shell: |
|
|
vault write database/roles/{{ service }}-app \
|
|
db_name="{{ service }}" \
|
|
creation_statements="CREATE ROLE \"{% raw %}{{name}}{% endraw %}\" WITH LOGIN PASSWORD '{% raw %}{{password}}{% endraw %}' VALID UNTIL '{% raw %}{{expiration}}{% endraw %}' INHERIT; GRANT {{ service }}_app TO \"{% raw %}{{name}}{% endraw %}\"; ALTER ROLE \"{% raw %}{{name}}{% endraw %}\" SET ROLE = {{ service }}_app;" \
|
|
revocation_statements="REASSIGN OWNED BY \"{% raw %}{{name}}{% endraw %}\" TO {{ service }}_owner; REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM \"{% raw %}{{name}}{% endraw %}\"; REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM \"{% raw %}{{name}}{% endraw %}\"; REVOKE USAGE ON SCHEMA public FROM \"{% raw %}{{name}}{% endraw %}\"; REVOKE CONNECT ON DATABASE {{ service }} FROM \"{% raw %}{{name}}{% endraw %}\"; DROP ROLE IF EXISTS \"{% raw %}{{name}}{% endraw %}\";" \
|
|
default_ttl="1h" \
|
|
max_ttl="24h"
|
|
when: "'app' in (svc.vault_roles | default(['app', 'migrate']))"
|
|
|
|
- name: Create Vault migrate role
|
|
shell: |
|
|
vault write database/roles/{{ service }}-migrate \
|
|
db_name="{{ service }}" \
|
|
creation_statements="CREATE ROLE \"{% raw %}{{name}}{% endraw %}\" WITH LOGIN PASSWORD '{% raw %}{{password}}{% endraw %}' VALID UNTIL '{% raw %}{{expiration}}{% endraw %}' INHERIT; GRANT {{ service }}_migrate TO \"{% raw %}{{name}}{% endraw %}\"; ALTER ROLE \"{% raw %}{{name}}{% endraw %}\" SET ROLE = {{ service }}_migrate;" \
|
|
revocation_statements="REASSIGN OWNED BY \"{% raw %}{{name}}{% endraw %}\" TO {{ service }}_owner; REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM \"{% raw %}{{name}}{% endraw %}\"; REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM \"{% raw %}{{name}}{% endraw %}\"; REVOKE ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public FROM \"{% raw %}{{name}}{% endraw %}\"; REVOKE ALL PRIVILEGES ON SCHEMA public FROM \"{% raw %}{{name}}{% endraw %}\"; REVOKE CONNECT ON DATABASE {{ service }} FROM \"{% raw %}{{name}}{% endraw %}\"; DROP ROLE IF EXISTS \"{% raw %}{{name}}{% endraw %}\";" \
|
|
default_ttl="15m" \
|
|
max_ttl="1h"
|
|
when: "'migrate' in (svc.vault_roles | default(['app', 'migrate']))"
|
|
|
|
- name: Display Service Summary
|
|
hosts: localhost
|
|
gather_facts: false
|
|
vars_files:
|
|
- ../services.yml
|
|
vars:
|
|
svc: "{{ services[service] }}"
|
|
postgres_ip: "{{ hostvars['postgres-01']['nebula_ip'] }}"
|
|
valkey_ip: "{{ hostvars['valkey-01']['nebula_ip'] }}"
|
|
garage_ip: "{{ hostvars['garage-01']['nebula_ip'] }}"
|
|
tasks:
|
|
- name: Service provisioning complete
|
|
debug:
|
|
msg:
|
|
- "=========================================="
|
|
- "Service: {{ service }}"
|
|
- "Description: {{ svc.description }}"
|
|
- "=========================================="
|
|
- ""
|
|
- "PostgreSQL:"
|
|
- " Database: {{ service }} @ {{ postgres_ip }}:5432"
|
|
- " App credentials: vault read database/creds/{{ service }}-app"
|
|
- " Migrate credentials: vault read database/creds/{{ service }}-migrate"
|
|
- ""
|
|
- "Valkey:"
|
|
- " Host: {{ valkey_ip }}:6379"
|
|
- " User: {{ service }}"
|
|
- " Key prefix: {{ svc.valkey.key_prefix | default(service) }}:*"
|
|
- " Credentials: vault kv get secret/{{ service }}/valkey"
|
|
- ""
|
|
- "S3:"
|
|
- " Bucket: {{ svc.s3.bucket | default(service + '-media') }} @ http://{{ garage_ip }}:3900"
|
|
- " Credentials: vault kv get secret/{{ service }}/s3"
|
|
- ""
|
|
- "=========================================="
|