diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index da3c4baaf..9923ff023 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -62,8 +62,6 @@ podAnnotations: extraEnvs: [] # Exemple of settings to make snapshot view working in the ui when using AWS - # - name: WALE_S3_ENDPOINT - # value: https+path://s3.us-east-1.amazonaws.com:443 # - name: SPILO_S3_BACKUP_PREFIX # value: spilo/ # - name: AWS_ACCESS_KEY_ID @@ -83,8 +81,6 @@ extraEnvs: # key: AWS_DEFAULT_REGION # - name: SPILO_S3_BACKUP_BUCKET # value: - # - name: "USE_AWS_INSTANCE_PROFILE" - # value: "true" # configure UI service service: diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index e09dd1e4f..3b3097416 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -81,8 +81,6 @@ spec: ] } # Exemple of settings to make snapshot view working in the ui when using AWS - # - name: WALE_S3_ENDPOINT - # value: https+path://s3.us-east-1.amazonaws.com:443 # - name: SPILO_S3_BACKUP_PREFIX # value: spilo/ # - name: AWS_ACCESS_KEY_ID @@ -102,5 +100,3 @@ spec: # key: AWS_DEFAULT_REGION # - name: SPILO_S3_BACKUP_BUCKET # value: - # - name: "USE_AWS_INSTANCE_PROFILE" - # value: "true" diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index e02c2995c..bf28df6eb 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -95,14 +95,6 @@ DEFAULT_CPU = getenv('DEFAULT_CPU', '10m') DEFAULT_CPU_LIMIT = getenv('DEFAULT_CPU_LIMIT', '300m') -WALE_S3_ENDPOINT = getenv( - 'WALE_S3_ENDPOINT', - 'https+path://s3.eu-central-1.amazonaws.com:443', -) - -USE_AWS_INSTANCE_PROFILE = ( - getenv('USE_AWS_INSTANCE_PROFILE', 'false').lower() != 'false' -) AWS_ENDPOINT = getenv('AWS_ENDPOINT') @@ -784,8 +776,6 @@ def get_versions(pg_cluster: str): bucket=SPILO_S3_BACKUP_BUCKET, pg_cluster=pg_cluster, prefix=SPILO_S3_BACKUP_PREFIX, - s3_endpoint=WALE_S3_ENDPOINT, - use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE, ), ) @@ -797,9 +787,8 @@ def get_basebackups(pg_cluster: str, uid: str): bucket=SPILO_S3_BACKUP_BUCKET, pg_cluster=pg_cluster, prefix=SPILO_S3_BACKUP_PREFIX, - s3_endpoint=WALE_S3_ENDPOINT, uid=uid, - use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE, + postgresql_versions=OPERATOR_UI_CONFIG.get('postgresql_versions', DEFAULT_UI_CONFIG['postgresql_versions']), ), ) @@ -991,8 +980,6 @@ def main(port, debug, clusters: list): logger.info(f'Superuser team: {SUPERUSER_TEAM}') logger.info(f'Target namespace: {TARGET_NAMESPACE}') logger.info(f'Teamservice URL: {TEAM_SERVICE_URL}') - logger.info(f'Use AWS instance_profile: {USE_AWS_INSTANCE_PROFILE}') - logger.info(f'WAL-E S3 endpoint: {WALE_S3_ENDPOINT}') logger.info(f'AWS S3 endpoint: {AWS_ENDPOINT}') if TARGET_NAMESPACE is None: diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index f715430a1..6a2f03bb2 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -6,9 +6,8 @@ from requests import Session from urllib.parse import urljoin from uuid import UUID -from wal_e.cmd import configure_backup_cxt -from .utils import Attrs, defaulting, these +from .utils import defaulting, these from operator_ui.adapters.logger import logger session = Session() @@ -284,10 +283,8 @@ def read_stored_clusters(bucket, prefix, delimiter='/'): def read_versions( pg_cluster, bucket, - s3_endpoint, prefix, delimiter='/', - use_aws_instance_profile=False, ): return [ 'base' if uid == 'wal' else uid @@ -305,35 +302,72 @@ def read_versions( if uid == 'wal' or defaulting(lambda: UUID(uid)) ] -BACKUP_VERSION_PREFIXES = ['', '10/', '11/', '12/', '13/', '14/', '15/', '16/', '17/'] +def lsn_to_wal_segment_stop(finish_lsn, start_segment, wal_segment_size=16 * 1024 * 1024): + timeline = int(start_segment[:8], 16) + log_id = finish_lsn >> 32 + seg_id = (finish_lsn & 0xFFFFFFFF) // wal_segment_size + return f"{timeline:08X}{log_id:08X}{seg_id:08X}" + +def lsn_to_offset_hex(lsn, wal_segment_size=16 * 1024 * 1024): + return f"{lsn % wal_segment_size:08X}" def read_basebackups( pg_cluster, uid, bucket, - s3_endpoint, prefix, - delimiter='/', - use_aws_instance_profile=False, + postgresql_versions, ): - environ['WALE_S3_ENDPOINT'] = s3_endpoint suffix = '' if uid == 'base' else '/' + uid backups = [] - for vp in BACKUP_VERSION_PREFIXES: - - backups = backups + [ - { - key: value - for key, value in basebackup.__dict__.items() - if isinstance(value, str) or isinstance(value, int) - } - for basebackup in Attrs.call( - f=configure_backup_cxt, - aws_instance_profile=use_aws_instance_profile, - s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/{vp}', - )._backup_list(detail=True) - ] + for vp in postgresql_versions: + backup_prefix = f'{prefix}{pg_cluster}{suffix}/wal/{vp}/basebackups_005/' + logger.info(f"{bucket}/{backup_prefix}") + + paginator = client('s3').get_paginator('list_objects_v2') + pages = paginator.paginate(Bucket=bucket, Prefix=backup_prefix) + + for page in pages: + for obj in page.get("Contents", []): + key = obj["Key"] + if not key.endswith("backup_stop_sentinel.json"): + continue + + response = client('s3').get_object(Bucket=bucket, Key=key) + backup_info = loads(response["Body"].read().decode("utf-8")) + last_modified = response["LastModified"].astimezone(timezone.utc).isoformat() + + backup_name = key.split("/")[-1].replace("_backup_stop_sentinel.json", "") + start_seg, start_offset = backup_name.split("_")[1], backup_name.split("_")[-1] if "_" in backup_name else None + + if "LSN" in backup_info and "FinishLSN" in backup_info: + # WAL-G + lsn = backup_info["LSN"] + finish_lsn = backup_info["FinishLSN"] + backups.append({ + "expanded_size_bytes": backup_info.get("UncompressedSize"), + "last_modified": last_modified, + "name": backup_name, + "wal_segment_backup_start": start_seg, + "wal_segment_backup_stop": lsn_to_wal_segment_stop(finish_lsn, start_seg), + "wal_segment_offset_backup_start": lsn_to_offset_hex(lsn), + "wal_segment_offset_backup_stop": lsn_to_offset_hex(finish_lsn), + }) + elif "wal_segment_backup_stop" in backup_info: + # WAL-E + stop_seg = backup_info["wal_segment_backup_stop"] + stop_offset = backup_info["wal_segment_offset_backup_stop"] + + backups.append({ + "expanded_size_bytes": backup_info.get("expanded_size_bytes"), + "last_modified": last_modified, + "name": backup_name, + "wal_segment_backup_start": start_seg, + "wal_segment_backup_stop": stop_seg, + "wal_segment_offset_backup_start": start_offset, + "wal_segment_offset_backup_stop": stop_offset, + }) return backups diff --git a/ui/requirements.txt b/ui/requirements.txt index d3318ceec..783c0aac3 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -11,5 +11,4 @@ kubernetes==11.0.0 python-json-logger==2.0.7 requests==2.32.2 stups-tokens>=1.1.19 -wal_e==1.1.1 werkzeug==3.0.6