diff --git a/config_restore.go b/config_restore.go index 7194a30e..397c460f 100644 --- a/config_restore.go +++ b/config_restore.go @@ -18,6 +18,7 @@ import ( "fmt" a "github.com/aerospike/aerospike-client-go/v7" + "github.com/aerospike/backup-go/models" ) // RestoreConfig contains configuration for the restore operation. @@ -35,6 +36,9 @@ type RestoreConfig struct { EncryptionPolicy *EncryptionPolicy // Compression details. CompressionPolicy *CompressionPolicy + // Configuration of retries for each restore write operation. + // If nil, no retries will be performed. + RetryPolicy *models.RetryPolicy // Secret agent config. SecretAgentConfig *SecretAgentConfig // The sets to restore (optional, given an empty list, all sets will be restored). diff --git a/go.mod b/go.mod index d7a0b30b..34ca3955 100644 --- a/go.mod +++ b/go.mod @@ -3,18 +3,18 @@ module github.com/aerospike/backup-go go 1.21 require ( - github.com/aerospike/aerospike-client-go/v7 v7.6.0 + github.com/aerospike/aerospike-client-go/v7 v7.6.1 github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 - github.com/aws/aws-sdk-go-v2 v1.30.3 - github.com/aws/aws-sdk-go-v2/config v1.27.27 - github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 - github.com/aws/smithy-go v1.20.3 - github.com/docker/docker v27.1.1+incompatible + github.com/aws/aws-sdk-go-v2 v1.30.4 + github.com/aws/aws-sdk-go-v2/config v1.27.28 + github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0 + github.com/aws/smithy-go v1.20.4 + github.com/docker/docker v27.1.2+incompatible github.com/docker/go-connections v0.5.0 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/klauspost/compress v1.17.9 - github.com/minio/minio-go/v7 v7.0.74 + github.com/minio/minio-go/v7 v7.0.75 github.com/stretchr/testify v1.9.0 golang.org/x/sync v0.8.0 golang.org/x/time v0.6.0 @@ -23,20 +23,20 @@ require ( require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.28 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.4 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect diff --git a/go.sum b/go.sum index b13520c7..611fc319 100644 --- a/go.sum +++ b/go.sum @@ -2,46 +2,46 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/aerospike/aerospike-client-go/v7 v7.6.0 h1:jAOlsxOaWbmtGzB1yP9x1komh4x14BvCb5HRu5AzVIo= -github.com/aerospike/aerospike-client-go/v7 v7.6.0/go.mod h1:uCbSYMpjlRcH/9f26VSF/luzDDXrcDaV8c6/WIcKtT4= +github.com/aerospike/aerospike-client-go/v7 v7.6.1 h1:VZK6S9YKq2w6ptTk3kXXjTxG2U9M9Y7Oi3YQ+3T7wQQ= +github.com/aerospike/aerospike-client-go/v7 v7.6.1/go.mod h1:uCbSYMpjlRcH/9f26VSF/luzDDXrcDaV8c6/WIcKtT4= github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 h1:tOLqcGsc6A656WNEZhYZYxDiX7d6wCkEN6+jLDSDeUU= github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= -github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= -github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM= -github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= -github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= -github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 h1:Z5r7SycxmSllHYmaAZPpmN8GviDrSGhMS6bldqtXZPw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15/go.mod h1:CetW7bDE00QoGEmPUoZuRog07SGVAUVW6LFpNP0YfIg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 h1:YPYe6ZmvUfDDDELqEKtAd6bo8zxhkm+XEFEzQisqUIE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17/go.mod h1:oBtcnYua/CgzCWYN7NZ5j7PotFDaFSUjCYVTtfyn7vw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 h1:hT8ZAZRIfqBqHbzKTII+CIiY8G2oC9OpLedkZ51DWl8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= -github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= -github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= +github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw= +github.com/aws/aws-sdk-go-v2/config v1.27.28 h1:OTxWGW/91C61QlneCtnD62NLb4W616/NM1jA8LhJqbg= +github.com/aws/aws-sdk-go-v2/config v1.27.28/go.mod h1:uzVRVtJSU5EFv6Fu82AoVFKozJi2ZCY6WRCXj06rbvs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.28 h1:m8+AHY/ND8CMHJnPoH7PJIRakWGa4gbfbxuY9TGTUXM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.28/go.mod h1:6TF7dSc78ehD1SL6KpRIPKMA1GyyWflIkjqg+qmf4+c= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16/go.mod h1:Uyk1zE1VVdsHSU7096h/rwnXDzOzYQVl+FNPhPw7ShY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0 h1:Cso4Ev/XauMVsbwdhYEoxg8rxZWw43CFqqaPB5w3W2c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.4 h1:iAckBT2OeEK/kBDyN/jDtpEExhjeeA/Im2q4X0rJZT8= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.4/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= @@ -53,8 +53,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= -github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.1.2+incompatible h1:AhGzR1xaQIy53qCkxARaFluI00WPGtXn0AJuoQsVYTY= +github.com/docker/docker v27.1.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -97,8 +97,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= -github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= +github.com/minio/minio-go/v7 v7.0.75 h1:0uLrB6u6teY2Jt+cJUVi9cTvDRuBKWSRzSAcznRkwlE= +github.com/minio/minio-go/v7 v7.0.75/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= diff --git a/handler_restore.go b/handler_restore.go index 381b80ff..e77993c3 100644 --- a/handler_restore.go +++ b/handler_restore.go @@ -291,6 +291,7 @@ func (rh *RestoreHandler) runRestoreBatch( rh.logger, useBatchWrites, rh.config.BatchSize, + rh.config.RetryPolicy, ) statsWriter := newWriterWithTokenStats(writer, &rh.stats, rh.logger) diff --git a/io/aerospike/record_batch_writer.go b/io/aerospike/record_batch_writer.go index fe5e48b9..7c58dcf7 100644 --- a/io/aerospike/record_batch_writer.go +++ b/io/aerospike/record_batch_writer.go @@ -15,6 +15,7 @@ package aerospike import ( + "fmt" "log/slog" a "github.com/aerospike/aerospike-client-go/v7" @@ -27,6 +28,7 @@ type batchRecordWriter struct { writePolicy *a.WritePolicy stats *models.RestoreStats logger *slog.Logger + retryPolicy *models.RetryPolicy operationBuffer []a.BatchRecordIfc batchSize int } @@ -35,7 +37,7 @@ func (rw *batchRecordWriter) writeRecord(record *models.Record) error { writeOp := rw.batchWrite(record) rw.operationBuffer = append(rw.operationBuffer, writeOp) - if len(rw.operationBuffer) > rw.batchSize { + if len(rw.operationBuffer) >= rw.batchSize { return rw.flushBuffer() } @@ -76,28 +78,87 @@ func (rw *batchRecordWriter) close() error { func (rw *batchRecordWriter) flushBuffer() error { if len(rw.operationBuffer) == 0 { + rw.logger.Debug("Flush empty buffer") return nil } - err := rw.asc.BatchOperate(nil, rw.operationBuffer) - if err != nil { - if !err.Matches(atypes.GENERATION_ERROR, atypes.KEY_EXISTS_ERROR) { - return err + var ( + attempt uint + err a.Error + ) + + rw.logger.Debug("Starting batch operation", + slog.Int("bufferSize", len(rw.operationBuffer)), + slog.Any("retryPolicy", rw.retryPolicy), + ) + + for { + rw.logger.Debug("Attempting batch operation", + slog.Any("attempt", attempt), + slog.Int("bufferSize", len(rw.operationBuffer)), + ) + + err = rw.asc.BatchOperate(nil, rw.operationBuffer) + + if isNilOrAcceptableError(err) { + rw.operationBuffer = rw.processAndFilterOperations() + if len(rw.operationBuffer) == 0 { + rw.logger.Debug("All operations succeeded") + return nil + } + } else if !shouldRetry(err) { + return fmt.Errorf("non-retryable error on restore: %w", err) } + + attempt++ + + if !attemptsLeft(rw.retryPolicy, attempt) { + break + } + + rw.logger.Debug("Retryable error occurred", + slog.Any("error", err), + slog.Int("remainingOperations", len(rw.operationBuffer)), + ) + + sleep(rw.retryPolicy, attempt) } - for i := 0; i < len(rw.operationBuffer); i++ { - switch rw.operationBuffer[i].BatchRec().ResultCode { - case atypes.OK: - rw.stats.IncrRecordsInserted() - case atypes.GENERATION_ERROR: - rw.stats.IncrRecordsFresher() - case atypes.KEY_EXISTS_ERROR: - rw.stats.IncrRecordsExisted() + rw.logger.Error("Max retries reached", + slog.Any("attempts", attempt), + slog.Int("failedOperations", len(rw.operationBuffer)), + slog.Any("lastError", err), + ) + + return fmt.Errorf("max retries reached, %d operations failed: %w", len(rw.operationBuffer), err) +} +func (rw *batchRecordWriter) processAndFilterOperations() []a.BatchRecordIfc { + failedOps := make([]a.BatchRecordIfc, 0) + + for _, op := range rw.operationBuffer { + if rw.processOperationResult(op) { + failedOps = append(failedOps, op) } } - rw.operationBuffer = nil + return failedOps +} - return nil +// processOperationResult increases statistics counters. +// it returns true if operation should be retried. +func (rw *batchRecordWriter) processOperationResult(op a.BatchRecordIfc) bool { + code := op.BatchRec().ResultCode + switch code { + case atypes.OK: + rw.stats.IncrRecordsInserted() + return false + case atypes.GENERATION_ERROR: + rw.stats.IncrRecordsFresher() + return false + case atypes.KEY_EXISTS_ERROR: + rw.stats.IncrRecordsExisted() + return false + default: + return true + } } diff --git a/io/aerospike/record_writer.go b/io/aerospike/record_writer.go index 35406294..8106ff63 100644 --- a/io/aerospike/record_writer.go +++ b/io/aerospike/record_writer.go @@ -26,6 +26,7 @@ type singleRecordWriter struct { asc dbWriter writePolicy *a.WritePolicy stats *models.RestoreStats + retryPolicy *models.RetryPolicy } func (rw *singleRecordWriter) writeRecord(record *models.Record) error { @@ -36,24 +37,51 @@ func (rw *singleRecordWriter) writeRecord(record *models.Record) error { writePolicy = &setGenerationPolicy } - aerr := rw.asc.Put(writePolicy, record.Key, record.Bins) - if aerr != nil { - if aerr.Matches(atypes.GENERATION_ERROR) { - rw.stats.IncrRecordsFresher() + err := rw.executeWrite(writePolicy, record) + if err != nil { + return fmt.Errorf("error writing record %s: %w", record.Key.Digest(), err) + } + + return nil +} + +func (rw *singleRecordWriter) executeWrite(writePolicy *a.WritePolicy, record *models.Record) error { + var ( + aerr a.Error + attempt uint + ) + + for attemptsLeft(rw.retryPolicy, attempt) { + aerr = rw.asc.Put(writePolicy, record.Key, record.Bins) + if aerr == nil { + rw.stats.IncrRecordsInserted() + return nil } - if aerr.Matches(atypes.KEY_EXISTS_ERROR) { - rw.stats.IncrRecordsExisted() + if isNilOrAcceptableError(aerr) { + switch { + case aerr.Matches(atypes.GENERATION_ERROR): + rw.stats.IncrRecordsFresher() + case aerr.Matches(atypes.KEY_EXISTS_ERROR): + rw.stats.IncrRecordsExisted() + } + return nil } - return fmt.Errorf("error writing record %s: %w", record.Key.Digest(), aerr) - } + if shouldRetry(aerr) { + sleep(rw.retryPolicy, attempt) - rw.stats.IncrRecordsInserted() + attempt++ - return nil + continue + } + + return aerr + } + + return aerr } func (rw *singleRecordWriter) close() error { diff --git a/io/aerospike/record_writer_test.go b/io/aerospike/record_writer_test.go index 86283efd..b2efd42f 100644 --- a/io/aerospike/record_writer_test.go +++ b/io/aerospike/record_writer_test.go @@ -17,10 +17,13 @@ package aerospike import ( "log/slog" "testing" + "time" a "github.com/aerospike/aerospike-client-go/v7" + "github.com/aerospike/aerospike-client-go/v7/types" "github.com/aerospike/backup-go/io/aerospike/mocks" "github.com/aerospike/backup-go/models" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -56,7 +59,7 @@ func (suite *writersTestSuite) TestRestoreWriterRecord() { mockDBWriter.EXPECT().Put(policy, expRecord.Key, expRecord.Bins).Return(nil) stats := &models.RestoreStats{} - writer := newRecordWriter(mockDBWriter, policy, stats, slog.Default(), false, 1) + writer := newRecordWriter(mockDBWriter, policy, stats, slog.Default(), false, 1, nil) suite.NotNil(writer) err := writer.writeRecord(&expRecord) @@ -73,7 +76,7 @@ func (suite *writersTestSuite) TestRestoreWriterRecordFail() { mockDBWriter := mocks.NewMockdbWriter(suite.T()) policy := &a.WritePolicy{} stats := &models.RestoreStats{} - writer := newRecordWriter(mockDBWriter, policy, stats, slog.Default(), false, 1) + writer := newRecordWriter(mockDBWriter, policy, stats, slog.Default(), false, 1, nil) rec := models.Record{ Record: &a.Record{ Key: key, @@ -116,7 +119,7 @@ func (suite *writersTestSuite) TestRestoreWriterWithPolicy() { mockDBWriter.EXPECT().Put(policy, expRecord.Key, expRecord.Bins).Return(nil) stats := &models.RestoreStats{} - writer := newRecordWriter(mockDBWriter, policy, stats, slog.Default(), false, 1) + writer := newRecordWriter(mockDBWriter, policy, stats, slog.Default(), false, 1, nil) suite.NotNil(writer) err := writer.writeRecord(&expRecord) @@ -124,3 +127,88 @@ func (suite *writersTestSuite) TestRestoreWriterWithPolicy() { suite.Nil(err) suite.Equal(1, int(stats.GetRecordsInserted())) } + +func (suite *writersTestSuite) TestSingleRecordWriterRetry() { + namespace := "test" + set := "" + key, _ := a.NewKey(namespace, set, "key") + mockDBWriter := mocks.NewMockdbWriter(suite.T()) + policy := &a.WritePolicy{} + stats := &models.RestoreStats{} + retryPolicy := &models.RetryPolicy{ + BaseTimeout: 10 * time.Millisecond, + Multiplier: 1, + MaxRetries: 2, + } + writer := newRecordWriter(mockDBWriter, policy, stats, slog.Default(), false, 1, retryPolicy) + rec := models.Record{ + Record: &a.Record{ + Key: key, + Bins: a.BinMap{ + "key0": "hi", + "key1": 1, + }, + }, + } + + mockDBWriter.On("Put", policy, rec.Key, rec.Bins). + Return(a.ErrConnectionPoolEmpty).Once() + mockDBWriter.On("Put", policy, rec.Key, rec.Bins). + Return(a.ErrTimeout).Once() + mockDBWriter.On("Put", policy, rec.Key, rec.Bins). + Return(nil).Once() + + err := writer.writeRecord(&rec) + suite.Nil(err) + + err = writer.close() + suite.Nil(err) + suite.Equal(1, int(stats.GetRecordsInserted())) + + mockDBWriter.AssertExpectations(suite.T()) +} + +func (suite *writersTestSuite) TestBatchRecordWriterRetry() { + namespace := "test" + set := "" + key, _ := a.NewKey(namespace, set, "key") + mockDBWriter := mocks.NewMockdbWriter(suite.T()) + policy := &a.WritePolicy{} + stats := &models.RestoreStats{} + retryPolicy := &models.RetryPolicy{ + BaseTimeout: 10 * time.Millisecond, + Multiplier: 1, + MaxRetries: 2, + } + writer := newRecordWriter(mockDBWriter, policy, stats, slog.Default(), true, 1, retryPolicy) + rec := models.Record{ + Record: &a.Record{ + Key: key, + Bins: a.BinMap{ + "key0": "hi", + "key1": 1, + }, + }, + } + + mockDBWriter.On("BatchOperate", mock.Anything, mock.Anything). + Return(a.ErrConnectionPoolEmpty).Once() + mockDBWriter.On("BatchOperate", mock.Anything, mock.Anything). + Return(a.ErrTimeout).Once() + mockDBWriter.On("BatchOperate", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + batchOps := args.Get(1).([]a.BatchRecordIfc) + for _, op := range batchOps { + op.BatchRec().ResultCode = types.OK + } + }).Return(nil).Once() + + err := writer.writeRecord(&rec) + suite.Nil(err) + + err = writer.close() + suite.Nil(err) + suite.Equal(1, int(stats.GetRecordsInserted())) + + mockDBWriter.AssertExpectations(suite.T()) +} diff --git a/io/aerospike/restore_writer.go b/io/aerospike/restore_writer.go index f1562571..0ebb9a94 100644 --- a/io/aerospike/restore_writer.go +++ b/io/aerospike/restore_writer.go @@ -17,8 +17,11 @@ package aerospike import ( "errors" "log/slog" + "math" + "time" a "github.com/aerospike/aerospike-client-go/v7" + atypes "github.com/aerospike/aerospike-client-go/v7/types" "github.com/aerospike/backup-go/internal/logging" "github.com/aerospike/backup-go/models" "github.com/aerospike/backup-go/pipeline" @@ -42,7 +45,8 @@ type restoreWriter struct { // NewRestoreWriter creates a new restoreWriter. func NewRestoreWriter(asc dbWriter, writePolicy *a.WritePolicy, stats *models.RestoreStats, - logger *slog.Logger, useBatchWrites bool, batchSize int) pipeline.DataWriter[*models.Token] { + logger *slog.Logger, useBatchWrites bool, batchSize int, retryPolicy *models.RetryPolicy, +) pipeline.DataWriter[*models.Token] { logger = logging.WithWriter(logger, uuid.NewString(), logging.WriterTypeRestore) logger.Debug("created new restore writer") @@ -57,7 +61,7 @@ func NewRestoreWriter(asc dbWriter, writePolicy *a.WritePolicy, stats *models.Re writePolicy: writePolicy, logger: logger, }, - recordWriter: newRecordWriter(asc, writePolicy, stats, logger, useBatchWrites, batchSize), + recordWriter: newRecordWriter(asc, writePolicy, stats, logger, useBatchWrites, batchSize, retryPolicy), logger: logger, } } @@ -67,6 +71,7 @@ func newRecordWriter(asc dbWriter, writePolicy *a.WritePolicy, logger *slog.Logger, useBatchWrites bool, batchSize int, + retryPolicy *models.RetryPolicy, ) recordWriter { if useBatchWrites { return &batchRecordWriter{ @@ -75,6 +80,7 @@ func newRecordWriter(asc dbWriter, writePolicy *a.WritePolicy, stats: stats, logger: logger, batchSize: batchSize, + retryPolicy: retryPolicy, } } @@ -82,6 +88,7 @@ func newRecordWriter(asc dbWriter, writePolicy *a.WritePolicy, asc: asc, writePolicy: writePolicy, stats: stats, + retryPolicy: retryPolicy, } } @@ -103,6 +110,39 @@ func (rw *restoreWriter) Write(data *models.Token) (int, error) { // Close satisfies the DataWriter interface. func (rw *restoreWriter) Close() error { - rw.logger.Debug("closed restore writer") + rw.logger.Debug("close restore writer") return rw.close() } + +func attemptsLeft(rc *models.RetryPolicy, attempt uint) bool { + if rc == nil { + return attempt == 0 // only pass on 1st try. + } + + return attempt <= rc.MaxRetries +} + +func sleep(rc *models.RetryPolicy, attempt uint) { + if rc == nil { + return + } + + duration := time.Duration(float64(rc.BaseTimeout) * math.Pow(rc.Multiplier, float64(attempt))) + time.Sleep(duration) +} + +func isNilOrAcceptableError(err a.Error) bool { + return err == nil || err.Matches(atypes.GENERATION_ERROR, atypes.KEY_EXISTS_ERROR) +} + +func shouldRetry(err a.Error) bool { + return err != nil && err.Matches( + atypes.NO_AVAILABLE_CONNECTIONS_TO_NODE, + atypes.TIMEOUT, + atypes.DEVICE_OVERLOAD, + atypes.NETWORK_ERROR, + atypes.SERVER_NOT_AVAILABLE, + atypes.BATCH_FAILED, + atypes.MAX_ERROR_RATE, + ) +} diff --git a/models/retry_policy.go b/models/retry_policy.go new file mode 100644 index 00000000..a690973f --- /dev/null +++ b/models/retry_policy.go @@ -0,0 +1,31 @@ +// Copyright 2024 Aerospike, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import "time" + +// RetryPolicy defines the configuration for retry attempts in case of failures. +type RetryPolicy struct { + // BaseTimeout is the initial delay between retry attempts. + BaseTimeout time.Duration + + // Multiplier is used to increase the delay between subsequent retry attempts. + // The actual delay is calculated as: BaseTimeout * (Multiplier ^ attemptNumber) + Multiplier float64 + + // MaxRetries is the maximum number of retry attempts that will be made. + // If set to 0, no retries will be performed. + MaxRetries uint +}